<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/10/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/10/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8468.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8468.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 14.3–Scrapy 分布式实现</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="14-3-Scrapy-分布式实现"><a href="#14-3-Scrapy-分布式实现" class="headerlink" title="14.3 Scrapy 分布式实现"></a>14.3 Scrapy 分布式实现</h1>
                  <p>接下来，我们会利用 Scrapy-Redis 来实现分布式的对接。</p>
                  <h3 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h3>
                  <p>请确保已经成功实现了 Scrapy 新浪微博爬虫，Scrapy-Redis 库已经正确安装，如果还没安装，请参考第 1 章的安装说明。</p>
                  <h3 id="2-搭建-Redis-服务器"><a href="#2-搭建-Redis-服务器" class="headerlink" title="2. 搭建 Redis 服务器"></a>2. 搭建 Redis 服务器</h3>
                  <p>要实现分布式部署，多台主机需要共享爬取队列和去重集合，而这两部分内容都是存于 Redis 数据库中的，我们需要搭建一个可公网访问的 Redis 服务器。 推荐使用 Linux 服务器，可以购买阿里云、腾讯云、Azure 等提供的云主机，一般都会配有公网 IP，具体的搭建方式可以参考第 1 章中 Redis 数据库的安装方式。 Redis 安装完成之后就可以远程连接了，注意部分商家（如阿里云、腾讯云）的服务器需要配置安全组放通 Redis 运行端口才可以远程访问。如果遇到不能远程连接的问题，可以排查安全组的设置。 需要记录 Redis 的运行 IP、端口、地址，供后面配置分布式爬虫使用。当前配置好的 Redis 的 IP 为服务器的 IP 120.27.34.25，端口为默认的 6379，密码为 foobared。</p>
                  <h3 id="3-部署代理池和-Cookies-池"><a href="#3-部署代理池和-Cookies-池" class="headerlink" title="3. 部署代理池和 Cookies 池"></a>3. 部署代理池和 Cookies 池</h3>
                  <p>新浪微博项目需要用到代理池和 Cookies 池，而之前我们的代理池和 Cookies 池都是在本地运行的。所以我们需要将二者放到可以被公网访问的服务器上运行，将代码上传到服务器，修改 Redis 的连接信息配置，用同样的方式运行代理池和 Cookies 池。 远程访问代理池和 Cookies 池提供的接口，来获取随机代理和 Cookies。如果不能远程访问，先确保其在 0.0.0.0 这个 Host 上运行，再检查安全组的配置。 如我当前配置好的代理池和 Cookies 池的运行 IP 都是服务器的 IP，120.27.34.25，端口分别为 5555 和 5556，如图 14-3 和图 14-4 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-29-113501.jpg" alt=""> 图 14-3 代理池接口 <img src="https://qiniu.cuiqingcai.com/2019-11-29-113506.jpg" alt=""> 图 14-4 Cookies 池接口 所以接下来我们就需要把 Scrapy 新浪微博项目中的访问链接修改如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">PROXY_URL</span> = <span class="string">'http://120.27.34.25:5555/random'</span></span><br><span class="line"><span class="attr">COOKIES_URL</span> = <span class="string">'http://120.27.34.25:5556/weibo/random'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>具体的修改方式根据实际配置的 IP 和端口做相应调整。</p>
                  <h3 id="4-配置-Scrapy-Redis"><a href="#4-配置-Scrapy-Redis" class="headerlink" title="4. 配置 Scrapy-Redis"></a>4. 配置 Scrapy-Redis</h3>
                  <p>配置 Scrapy-Redis 非常简单，只需要修改一下 settings.py 配置文件即可。</p>
                  <h4 id="核心配置"><a href="#核心配置" class="headerlink" title="核心配置"></a>核心配置</h4>
                  <p>首先最主要的是，需要将调度器的类和去重的类替换为 Scrapy-Redis 提供的类，在 settings.py 里面添加如下配置即可：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SCHEDULER</span> = <span class="string">"scrapy_redis.scheduler.Scheduler"</span></span><br><span class="line"><span class="attr">DUPEFILTER_CLASS</span> = <span class="string">"scrapy_redis.dupefilter.RFPDupeFilter"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="Redis-连接配置"><a href="#Redis-连接配置" class="headerlink" title="Redis 连接配置"></a>Redis 连接配置</h4>
                  <p>接下来配置 Redis 的连接信息，这里有两种配置方式。 第一种方式是通过连接字符串配置。我们可以用 Redis 的地址、端口、密码来构造一个 Redis 连接字符串，支持的连接形式如下所示：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">redis:</span><span class="comment">//[:password]@host:port/db</span></span><br><span class="line"><span class="symbol">rediss:</span><span class="comment">//[:password]@host:port/db</span></span><br><span class="line"><span class="symbol">unix:</span><span class="comment">//[:password]@/path/to/socket.sock?db=db</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>password 是密码，比如要以冒号开头，中括号代表此选项可有可无，host 是 Redis 的地址，port 是运行端口，db 是数据库代号，其值默认是 0。 根据上文中提到我的 Redis 连接信息，构造这个 Redis 的连接字符串如下所示：</p>
                  <figure class="highlight avrasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">redis:</span>//:foobared<span class="subst">@120</span><span class="number">.27</span><span class="number">.34</span><span class="number">.25</span>:<span class="number">6379</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>直接在 settings.py 里面配置为 REDIS_URL 变量即可：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">REDIS_URL</span> = <span class="string">'redis://:foobared@120.27.34.25:6379'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第二种配置方式是分项单独配置。这个配置就更加直观明了，如根据我的 Redis 连接信息，可以在 settings.py 中配置如下代码：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">REDIS_HOST</span> = <span class="string">'120.27.34.25'</span></span><br><span class="line"><span class="attr">REDIS_PORT</span> = <span class="number">6379</span></span><br><span class="line"><span class="attr">REDIS_PASSWORD</span> = <span class="string">'foobared'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这段代码分开配置了 Redis 的地址、端口和密码。 注意，如果配置了 REDIS_URL，那么 Scrapy-Redis 将优先使用 REDIS_URL 连接，会覆盖上面的三项配置。如果想要分项单独配置的话，请不要配置 REDIS_URL。 在本项目中，我选择的是配置 REDIS_URL。</p>
                  <h4 id="配置调度队列"><a href="#配置调度队列" class="headerlink" title="配置调度队列"></a>配置调度队列</h4>
                  <p>此项配置是可选的，默认使用 PriorityQueue。如果想要更改配置，可以配置 SCHEDULER_QUEUE_CLASS 变量，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SCHEDULER_QUEUE_CLASS</span> = <span class="string">'scrapy_redis.queue.PriorityQueue'</span></span><br><span class="line"><span class="attr">SCHEDULER_QUEUE_CLASS</span> = <span class="string">'scrapy_redis.queue.FifoQueue'</span></span><br><span class="line"><span class="attr">SCHEDULER_QUEUE_CLASS</span> = <span class="string">'scrapy_redis.queue.LifoQueue'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上三行任选其一配置，即可切换爬取队列的存储方式。 在本项目中不进行任何配置，我们使用默认配置。</p>
                  <h4 id="配置持久化"><a href="#配置持久化" class="headerlink" title="配置持久化"></a>配置持久化</h4>
                  <p>此配置是可选的，默认是 False。Scrapy-Redis 默认会在爬取全部完成后清空爬取队列和去重指纹集合。 如果不想自动清空爬取队列和去重指纹集合，可以增加如下配置：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SCHEDULER_PERSIST</span> = <span class="literal">True</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>将 SCHEDULER_PERSIST 设置为 True 之后，爬取队列和去重指纹集合不会在爬取完成后自动清空，如果不配置，默认是 False，即自动清空。 值得注意的是，如果强制中断爬虫的运行，爬取队列和去重指纹集合是不会自动清空的。 在本项目中不进行任何配置，我们使用默认配置。</p>
                  <h4 id="配置重爬"><a href="#配置重爬" class="headerlink" title="配置重爬"></a>配置重爬</h4>
                  <p>此配置是可选的，默认是 False。如果配置了持久化或者强制中断了爬虫，那么爬取队列和指纹集合不会被清空，爬虫重新启动之后就会接着上次爬取。如果想重新爬取，我们可以配置重爬的选项：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SCHEDULER_FLUSH_ON_START</span> = <span class="literal">True</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样将 SCHEDULER_FLUSH_ON_START 设置为 True 之后，爬虫每次启动时，爬取队列和指纹集合都会清空。所以要做分布式爬取，我们必须保证只能清空一次，否则每个爬虫任务在启动时都清空一次，就会把之前的爬取队列清空，势必会影响分布式爬取。 注意，此配置在单机爬取的时候比较方便，分布式爬取不常用此配置。 在本项目中不进行任何配置，我们使用默认配置。</p>
                  <h4 id="Pipeline-配置"><a href="#Pipeline-配置" class="headerlink" title="Pipeline 配置"></a>Pipeline 配置</h4>
                  <p>此配置是可选的，默认不启动 Pipeline。Scrapy-Redis 实现了一个存储到 Redis 的 Item Pipeline，启用了这个 Pipeline 的话，爬虫会把生成的 Item 存储到 Redis 数据库中。在数据量比较大的情况下，我们一般不会这么做。因为 Redis 是基于内存的，我们利用的是它处理速度快的特性，用它来做存储未免太浪费了，配置如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">ITEM_PIPELINES</span> = &#123;<span class="string">'scrapy_redis.pipelines.RedisPipeline'</span>: <span class="number">300</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本项目不进行任何配置，即不启动 Pipeline。 到此为止，Scrapy-Redis 的配置就完成了。有的选项我们没有配置，但是这些配置在其他 Scrapy 项目中可能用到，要根据具体情况而定。</p>
                  <h3 id="5-配置存储目标"><a href="#5-配置存储目标" class="headerlink" title="5. 配置存储目标"></a>5. 配置存储目标</h3>
                  <p>之前 Scrapy 新浪微博爬虫项目使用的存储是 MongoDB，而且 MongoDB 是本地运行的，即连接的是 localhost。但是，当爬虫程序分发到各台主机运行的时候，爬虫就会连接各自的的 MongoDB。所以我们需要在各台主机上都安装 MongoDB，这样有两个缺点：一是搭建 MongoDB 环境比较烦琐；二是这样各台主机的爬虫会把爬取结果分散存到各自主机上，不方便统一管理。 所以我们最好将存储目标存到同一个地方，例如都存到同一个 MongoDB 数据库中。我们可以在服务器上搭建一个 MongoDB 服务，或者直接购买 MongoDB 数据存储服务。 这里使用的就是服务器上搭建的的 MongoDB 服务，IP 仍然为 120.27.34.25，用户名为 admin，密码为 admin123。 修改配置 MONGO_URI 为如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MONGO_URI</span> = <span class="string">'mongodb://admin:admin123@120.27.34.25:27017'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>到此为止，我们就成功完成了 Scrapy 分布式爬虫的配置了。</p>
                  <h3 id="6-运行"><a href="#6-运行" class="headerlink" title="6. 运行"></a>6. 运行</h3>
                  <p>接下来将代码部署到各台主机上，记得每台主机都需要配好对应的 Python 环境。 每台主机上都执行如下命令，即可启动爬取：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl weibocn</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>每台主机启动了此命令之后，就会从配置的 Redis 数据库中调度 Request，做到爬取队列共享和指纹集合共享。同时每台主机占用各自的带宽和处理器，不会互相影响，爬取效率成倍提高。</p>
                  <h3 id="7-结果"><a href="#7-结果" class="headerlink" title="7. 结果"></a>7. 结果</h3>
                  <p>一段时间后，我们可以用 RedisDesktop 观察远程 Redis 数据库的信息。这里会出现两个 Key：一个叫作 weibocn:dupefilter，用来储存指纹；另一个叫作 weibocn:requests，即爬取队列，如图 14-5 和图 14-6 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-29-113521.jpg" alt=""> 图 14-5 去重指纹 <img src="/Users/maqian/Downloads/python3webspiderupload/assets/14-6.jpg" alt=""> 图 14-6 爬取队列 随着时间的推移，指纹集合会不断增长，爬取队列会动态变化，爬取的数据也会被储存到 MongoDB 数据库中。 至此 Scrapy 分布式的配置已全部完成。</p>
                  <h3 id="8-本节代码"><a href="#8-本节代码" class="headerlink" title="8. 本节代码"></a>8. 本节代码</h3>
                  <p>本节代码地址为：<a href="https://github.com/Python3WebSpider/Weibo/tree/distributed" target="_blank" rel="noopener">https://github.com/Python3WebSpider/Weibo/tree/distributed</a>，注意这里是 distributed 分支。</p>
                  <h3 id="9-结语"><a href="#9-结语" class="headerlink" title="9. 结语"></a>9. 结语</h3>
                  <p>本节通过对接 Scrapy-Redis 成功实现了分布式爬虫，但是部署还是有很多不方便的地方。另外，如果爬取量特别大的话，Redis 的内存也是个问题。在后文我们会继续了解相关优化方案。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-09 10:14:40" itemprop="dateCreated datePublished" datetime="2019-12-09T10:14:40+08:00">2019-12-09</time>
                </span>
                <span id="/8468.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 14.3–Scrapy 分布式实现" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8465.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8465.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 14.2–Scrapy-Redis 源码解析</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="14-2-Scrapy-Redis-源码解析"><a href="#14-2-Scrapy-Redis-源码解析" class="headerlink" title="14.2 Scrapy-Redis 源码解析"></a>14.2 Scrapy-Redis 源码解析</h1>
                  <p>Scrapy-Redis 库已经为我们提供了 Scrapy 分布式的队列、调度器、去重等功能，其 GitHub 地址为：<a href="https://github.com/rmax/scrapy-redis" target="_blank" rel="noopener">https://github.com/rmax/scrapy-redis</a>。 本节我们深入了解一下，利用 Redis 如何实现 Scrapy 分布式。</p>
                  <h3 id="1-获取源码"><a href="#1-获取源码" class="headerlink" title="1. 获取源码"></a>1. 获取源码</h3>
                  <p>可以把源码克隆下来，执行如下命令：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">git <span class="keyword">clone</span> <span class="title">https</span>://github.com/rmax/scrapy-redis.git</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>核心源码在 scrapy-redis/src/scrapy_redis 目录下。</p>
                  <h3 id="2-爬取队列"><a href="#2-爬取队列" class="headerlink" title="2. 爬取队列"></a>2. 爬取队列</h3>
                  <p>从爬取队列入手，看看它的具体实现。源码文件为 queue.py，它有三个队列的实现，首先它实现了一个父类 Base，提供一些基本方法和属性，如下所示：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Base</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="string">"""Per-spider base queue class"""</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, server, spider, key, serializer=None)</span>:</span></span><br><span class="line">        <span class="keyword">if</span> serializer <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            serializer = picklecompat</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> hasattr(serializer, <span class="string">'loads'</span>):</span><br><span class="line">            <span class="keyword">raise</span> TypeError(<span class="string">"serializer does not implement 'loads' function: % r"</span></span><br><span class="line">                            % serializer)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> hasattr(serializer, <span class="string">'dumps'</span>):</span><br><span class="line">            <span class="keyword">raise</span> TypeError(<span class="string">"serializer '% s' does not implement 'dumps' function: % r"</span></span><br><span class="line">                            % serializer)</span><br><span class="line">        self.server = server</span><br><span class="line">        self.spider = spider</span><br><span class="line">        self.key = key % &#123;<span class="string">'spider'</span>: spider.name&#125;</span><br><span class="line">        self.serializer = serializer</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_encode_request</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        obj = request_to_dict(request, self.spider)</span><br><span class="line">        <span class="keyword">return</span> self.serializer.dumps(obj)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_decode_request</span><span class="params">(self, encoded_request)</span>:</span></span><br><span class="line">        obj = self.serializer.loads(encoded_request)</span><br><span class="line">        <span class="keyword">return</span> request_from_dict(obj, self.spider)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Return the length of the queue"""</span></span><br><span class="line">        <span class="keyword">raise</span> NotImplementedError</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">push</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Push a request"""</span></span><br><span class="line">        <span class="keyword">raise</span> NotImplementedError</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pop</span><span class="params">(self, timeout=<span class="number">0</span>)</span>:</span></span><br><span class="line">        <span class="string">"""Pop a request"""</span></span><br><span class="line">        <span class="keyword">raise</span> NotImplementedError</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">clear</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Clear queue/stack"""</span></span><br><span class="line">        self.server.delete(self.key)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先看一下 <em>encode<em>request() 和 _decode_request() 方法，因为我们需要把一 个 Request 对象存储到数据库中，但数据库无法直接存储对象，所以需要将 Request 序列化转成字符串再存储，而这两个方法就分别是序列化和反序列化的操作，利用 pickle 库来实现，一般在调用 push() 将 Request 存入数据库时会调用 _encode_request() 方法进行序列化，在调用 pop() 取出 Request 的时候会调用 _decode_request() 进行反序列化。 在父类中 __len</em></em>()、push() 和 pop() 方法都是未实现的，会直接抛出 NotImplementedError，因此这个类是不能直接被使用的，所以必须要实现一个子类来重写这三个方法，而不同的子类就会有不同的实现，也就有着不同的功能。 那么接下来就需要定义一些子类来继承 Base 类，并重写这几个方法，那在源码中就有三个子类的实现，它们分别是 FifoQueue、PriorityQueue、LifoQueue，我们分别来看下它们的实现原理。 首先是 FifoQueue：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">FifoQueue</span><span class="params">(Base)</span>:</span></span><br><span class="line">    <span class="string">"""Per-spider FIFO queue"""</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Return the length of the queue"""</span></span><br><span class="line">        <span class="keyword">return</span> self.server.llen(self.key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">push</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Push a request"""</span></span><br><span class="line">        self.server.lpush(self.key, self._encode_request(request))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pop</span><span class="params">(self, timeout=<span class="number">0</span>)</span>:</span></span><br><span class="line">        <span class="string">"""Pop a request"""</span></span><br><span class="line">        <span class="keyword">if</span> timeout &gt; <span class="number">0</span>:</span><br><span class="line">            data = self.server.brpop(self.key, timeout)</span><br><span class="line">            <span class="keyword">if</span> isinstance(data, tuple):</span><br><span class="line">                data = data[<span class="number">1</span>]</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            data = self.server.rpop(self.key)</span><br><span class="line">        <span class="keyword">if</span> data:</span><br><span class="line">            <span class="keyword">return</span> self._decode_request(data)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这个类继承了 Base 类，并重写了 <strong>len</strong>()、push()、pop() 这三个方法，在这三个方法中都是对 server 对象的操作，而 server 对象就是一个 Redis 连接对象，我们可以直接调用其操作 Redis 的方法对数据库进行操作，可以看到这里的操作方法有 llen()、lpush()、rpop() 等，那这就代表此爬取队列是使用的 Redis 的列表，序列化后的 Request 会被存入列表中，就是列表的其中一个元素，<strong>len</strong>() 方法是获取列表的长度，push() 方法中调用了 lpush() 操作，这代表从列表左侧存入数据，pop() 方法中调用了 rpop() 操作，这代表从列表右侧取出数据。 所以 Request 在列表中的存取顺序是左侧进、右侧出，所以这是有序的进出，即先进先出，英文叫做 First Input First Output，也被简称作 Fifo，而此类的名称就叫做 FifoQueue。 另外还有一个与之相反的实现类，叫做 LifoQueue，实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">LifoQueue</span><span class="params">(Base)</span>:</span></span><br><span class="line">    <span class="string">"""Per-spider LIFO queue."""</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Return the length of the stack"""</span></span><br><span class="line">        <span class="keyword">return</span> self.server.llen(self.key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">push</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Push a request"""</span></span><br><span class="line">        self.server.lpush(self.key, self._encode_request(request))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pop</span><span class="params">(self, timeout=<span class="number">0</span>)</span>:</span></span><br><span class="line">        <span class="string">"""Pop a request"""</span></span><br><span class="line">        <span class="keyword">if</span> timeout &gt; <span class="number">0</span>:</span><br><span class="line">            data = self.server.blpop(self.key, timeout)</span><br><span class="line">            <span class="keyword">if</span> isinstance(data, tuple):</span><br><span class="line">                data = data[<span class="number">1</span>]</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            data = self.server.lpop(self.key)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> data:</span><br><span class="line">            <span class="keyword">return</span> self._decode_request(data)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>与 FifoQueue 不同的就是它的 pop() 方法，在这里使用的是 lpop() 操作，也就是从左侧出，而 push() 方法依然是使用的 lpush() 操作，是从左侧入。那么这样达到的效果就是先进后出、后进先出，英文叫做 Last In First Out，简称为 Lifo，而此类名称就叫做 LifoQueue。同时这个存取方式类似栈的操作，所以其实也可以称作 StackQueue。 另外在源码中还有一个子类实现，叫做 PriorityQueue，顾名思义，它叫做优先级队列，实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PriorityQueue</span><span class="params">(Base)</span>:</span></span><br><span class="line">    <span class="string">"""Per-spider priority queue abstraction using redis' sorted set"""</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__len__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Return the length of the queue"""</span></span><br><span class="line">        <span class="keyword">return</span> self.server.zcard(self.key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">push</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Push a request"""</span></span><br><span class="line">        data = self._encode_request(request)</span><br><span class="line">        score = -request.priority</span><br><span class="line">        self.server.execute_command(<span class="string">'ZADD'</span>, self.key, score, data)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pop</span><span class="params">(self, timeout=<span class="number">0</span>)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        Pop a request</span></span><br><span class="line"><span class="string">        timeout not support in this queue class</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        pipe = self.server.pipeline()</span><br><span class="line">        pipe.multi()</span><br><span class="line">        pipe.zrange(self.key, <span class="number">0</span>, <span class="number">0</span>).zremrangebyrank(self.key, <span class="number">0</span>, <span class="number">0</span>)</span><br><span class="line">        results, count = pipe.execute()</span><br><span class="line">        <span class="keyword">if</span> results:</span><br><span class="line">            <span class="keyword">return</span> self._decode_request(results[<span class="number">0</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们可以看到 <strong>len</strong>()、push()、pop() 方法中使用了 server 对象的 zcard()、zadd()、zrange() 操作，可以知道这里使用的存储结果是有序集合 Sorted Set，在这个集合中每个元素都可以设置一个分数，那么这个分数就代表优先级。 在 <strong>len</strong>() 方法里调用了 zcard() 操作，返回的就是有序集合的大小，也就是爬取队列的长度，在 push() 方法中调用了 zadd() 操作，就是向集合中添加元素，这里的分数指定成 Request 的优先级的相反数，因为分数低的会排在集合的前面，所以这里高优先级的 Request 就会存在集合的最前面。pop() 方法是首先调用了 zrange() 操作取出了集合的第一个元素，因为最高优先级的 Request 会存在集合最前面，所以第一个元素就是最高优先级的 Request，然后再调用 zremrangebyrank() 操作将这个元素删除，这样就完成了取出并删除的操作。 此队列是默认使用的队列，也就是爬取队列默认是使用有序集合来存储的。</p>
                  <h3 id="3-去重过滤"><a href="#3-去重过滤" class="headerlink" title="3. 去重过滤"></a>3. 去重过滤</h3>
                  <p>前面说过 Scrapy 的去重是利用集合来实现的，而在 Scrapy 分布式中的去重就需要利用共享的集合，那么这里使用的就是 Redis 中的集合数据结构。我们来看看去重类是怎样实现的，源码文件是 dupefilter.py，其内实现了一个 RFPDupeFilter 类，如下所示：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RFPDupeFilter</span><span class="params">(BaseDupeFilter)</span>:</span></span><br><span class="line">    <span class="string">"""Redis-based request duplicates filter.</span></span><br><span class="line"><span class="string">    This class can also be used with default Scrapy's scheduler.</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    logger = logger</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, server, key, debug=False)</span>:</span></span><br><span class="line">        <span class="string">"""Initialize the duplicates filter.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        server : redis.StrictRedis</span></span><br><span class="line"><span class="string">            The redis server instance.</span></span><br><span class="line"><span class="string">        key : str</span></span><br><span class="line"><span class="string">            Redis key Where to store fingerprints.</span></span><br><span class="line"><span class="string">        debug : bool, optional</span></span><br><span class="line"><span class="string">            Whether to log filtered requests.</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        self.server = server</span><br><span class="line">        self.key = key</span><br><span class="line">        self.debug = debug</span><br><span class="line">        self.logdupes = <span class="literal">True</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_settings</span><span class="params">(cls, settings)</span>:</span></span><br><span class="line">        <span class="string">"""Returns an instance from given settings.</span></span><br><span class="line"><span class="string">        This uses by default the key ``dupefilter:&lt;timestamp&gt;``. When using the</span></span><br><span class="line"><span class="string">        ``scrapy_redis.scheduler.Scheduler`` class, this method is not used as</span></span><br><span class="line"><span class="string">        it needs to pass the spider name in the key.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        settings : scrapy.settings.Settings</span></span><br><span class="line"><span class="string">        Returns</span></span><br><span class="line"><span class="string">        -------</span></span><br><span class="line"><span class="string">        RFPDupeFilter</span></span><br><span class="line"><span class="string">            A RFPDupeFilter instance.</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        server = get_redis_from_settings(settings)</span><br><span class="line">        key = defaults.DUPEFILTER_KEY % &#123;<span class="string">'timestamp'</span>: int(time.time())&#125;</span><br><span class="line">        debug = settings.getbool(<span class="string">'DUPEFILTER_DEBUG'</span>)</span><br><span class="line">        <span class="keyword">return</span> cls(server, key=key, debug=debug)</span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span>:</span></span><br><span class="line">        <span class="string">"""Returns instance from crawler.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        crawler : scrapy.crawler.Crawler</span></span><br><span class="line"><span class="string">        Returns</span></span><br><span class="line"><span class="string">        -------</span></span><br><span class="line"><span class="string">        RFPDupeFilter</span></span><br><span class="line"><span class="string">            Instance of RFPDupeFilter.</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> cls.from_settings(crawler.settings)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">request_seen</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Returns True if request was already seen.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        request : scrapy.http.Request</span></span><br><span class="line"><span class="string">        Returns</span></span><br><span class="line"><span class="string">        -------</span></span><br><span class="line"><span class="string">        bool</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        fp = self.request_fingerprint(request)</span><br><span class="line">        added = self.server.sadd(self.key, fp)</span><br><span class="line">        <span class="keyword">return</span> added == <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">request_fingerprint</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""Returns a fingerprint for a given request.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        request : scrapy.http.Request</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        Returns</span></span><br><span class="line"><span class="string">        -------</span></span><br><span class="line"><span class="string">        str</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> request_fingerprint(request)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close</span><span class="params">(self, reason=<span class="string">''</span>)</span>:</span></span><br><span class="line">        <span class="string">"""Delete data on close. Called by Scrapy's scheduler.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        reason : str, optional</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        self.clear()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">clear</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""Clears fingerprints data."""</span></span><br><span class="line">        self.server.delete(self.key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">log</span><span class="params">(self, request, spider)</span>:</span></span><br><span class="line">        <span class="string">"""Logs given request.</span></span><br><span class="line"><span class="string">        Parameters</span></span><br><span class="line"><span class="string">        ----------</span></span><br><span class="line"><span class="string">        request : scrapy.http.Request</span></span><br><span class="line"><span class="string">        spider : scrapy.spiders.Spider</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">if</span> self.debug:</span><br><span class="line">            msg = <span class="string">"Filtered duplicate request: %(request) s"</span></span><br><span class="line">            self.logger.debug(msg, &#123;<span class="string">'request'</span>: request&#125;, extra=&#123;<span class="string">'spider'</span>: spider&#125;)</span><br><span class="line">        <span class="keyword">elif</span> self.logdupes:</span><br><span class="line">            msg = (<span class="string">"Filtered duplicate request %(request) s"</span></span><br><span class="line">                   <span class="string">"- no more duplicates will be shown"</span></span><br><span class="line">                   <span class="string">"(see DUPEFILTER_DEBUG to show all duplicates)"</span>)</span><br><span class="line">            self.logger.debug(msg, &#123;<span class="string">'request'</span>: request&#125;, extra=&#123;<span class="string">'spider'</span>: spider&#125;)</span><br><span class="line">            self.logdupes = <span class="literal">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里同样实现了一个 request_seen() 方法，和 Scrapy 中的 request_seen() 方法实现极其类似。不过这里集合使用的是 server 对象的 sadd() 操作，也就是集合不再是一个简单数据结构了，而是直接换成了数据库的存储方式。 鉴别重复的方式还是使用指纹，指纹同样是依靠 request_fingerprint() 方法来获取的。获取指纹之后就直接向集合添加指纹，如果添加成功，说明这个指纹原本不存在于集合中，返回值 1。代码中最后的返回结果是判定添加结果是否为 0，如果刚才的返回值为 1，那这个判定结果就是 False，也就是不重复，否则判定为重复。 这样我们就成功利用 Redis 的集合完成了指纹的记录和重复的验证。</p>
                  <h3 id="4-调度器"><a href="#4-调度器" class="headerlink" title="4. 调度器"></a>4. 调度器</h3>
                  <p>Scrapy-Redis 还帮我们实现了配合 Queue、DupeFilter 使用的调度器 Scheduler，源文件名称是 scheduler.py。我们可以指定一些配置，如 SCHEDULER_FLUSH_ON_START 即是否在爬取开始的时候清空爬取队列，SCHEDULER_PERSIST 即是否在爬取结束后保持爬取队列不清除。我们可以在 settings.py 里自由配置，而此调度器很好地实现了对接。 接下来我们看看两个核心的存取方法，实现如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">enqueue_request</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> request.dont_filter <span class="keyword">and</span> <span class="keyword">self</span>.df.request_seen(request)<span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.df.log(request, <span class="keyword">self</span>.spider)</span><br><span class="line">        <span class="keyword">return</span> False</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">stats:</span></span><br><span class="line">        <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/enqueued/redis'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">    <span class="keyword">self</span>.queue.push(request)</span><br><span class="line">    <span class="keyword">return</span> True</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">next_request</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    block_pop_timeout = <span class="keyword">self</span>.idle_before_close</span><br><span class="line">    request = <span class="keyword">self</span>.queue.pop(block_pop_timeout)</span><br><span class="line">    <span class="keyword">if</span> request <span class="keyword">and</span> <span class="keyword">self</span>.<span class="symbol">stats:</span></span><br><span class="line">        <span class="keyword">self</span>.stats.inc_value(<span class="string">'scheduler/dequeued/redis'</span>, spider=<span class="keyword">self</span>.spider)</span><br><span class="line">    <span class="keyword">return</span> request</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>enqueue_request() 可以向队列中添加 Request，核心操作就是调用 Queue 的 push() 操作，还有一些统计和日志操作。next_request() 就是从队列中取 Request，核心操作就是调用 Queue 的 pop() 操作，此时如果队列中还有 Request，则 Request 会直接取出来，爬取继续，否则如果队列为空，爬取则会重新开始。</p>
                  <h3 id="5-总结"><a href="#5-总结" class="headerlink" title="5. 总结"></a>5. 总结</h3>
                  <p>那么到现在为止我们就把之前所说的三个分布式的问题解决了，总结如下：</p>
                  <ul>
                    <li>爬取队列的实现，在这里提供了三种队列，使用了 Redis 的列表或有序集合来维护。</li>
                    <li>去重的实现，使用了 Redis 的集合来保存 Request 的指纹来提供重复过滤。</li>
                    <li>中断后重新爬取的实现，中断后 Redis 的队列没有清空，再次启动时调度器的 next_request() 会从队列中取到下一个 Request，继续爬取。</li>
                  </ul>
                  <h3 id="6-结语"><a href="#6-结语" class="headerlink" title="6. 结语"></a>6. 结语</h3>
                  <p>以上内容便是 Scrapy-Redis 的核心源码解析。Scrapy-Redis 中还提供了 Spider、Item Pipeline 的实现，不过它们并不是必须使用。 在下一节，我们会将 Scrapy-Redis 集成到之前所实现的 Scrapy 新浪微博项目中，实现多台主机协同爬取。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-09 10:08:42" itemprop="dateCreated datePublished" datetime="2019-12-09T10:08:42+08:00">2019-12-09</time>
                </span>
                <span id="/8465.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 14.2–Scrapy-Redis 源码解析" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8456.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8456.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 14.1–分布式爬虫原理</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="14-1-分布式爬虫原理"><a href="#14-1-分布式爬虫原理" class="headerlink" title="14.1 分布式爬虫原理"></a>14.1 分布式爬虫原理</h1>
                  <p>我们在前面已经实现了 Scrapy 微博爬虫，虽然爬虫是异步加多线程的，但是我们只能在一台主机上运行，所以爬取效率还是有限的，分布式爬虫则是将多台主机组合起来，共同完成一个爬取任务，这将大大提高爬取的效率。</p>
                  <h3 id="1-分布式爬虫架构"><a href="#1-分布式爬虫架构" class="headerlink" title="1. 分布式爬虫架构"></a>1. 分布式爬虫架构</h3>
                  <p>在了解分布式爬虫架构之前，首先回顾一下 Scrapy 的架构，如图 13-1 所示。 Scrapy 单机爬虫中有一个本地爬取队列 Queue，这个队列是利用 deque 模块实现的。如果新的 Request 生成就会放到队列里面，随后 Request 被 Scheduler 调度。之后，Request 交给 Downloader 执行爬取，简单的调度架构如图 14-1 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-29-113355.jpg" alt=""> 图 14-1 调度架构 如果两个 Scheduler 同时从队列里面取 Request，每个 Scheduler 都有其对应的 Downloader，那么在带宽足够、正常爬取且不考虑队列存取压力的情况下，爬取效率会有什么变化？没错，爬取效率会翻倍。 这样，Scheduler 可以扩展多个，Downloader 也可以扩展多个。而爬取队列 Queue 必须始终为一个，也就是所谓的共享爬取队列。这样才能保证 Scheduer 从队列里调度某个 Request 之后，其他 Scheduler 不会重复调度此 Request，就可以做到多个 Schduler 同步爬取。这就是分布式爬虫的基本雏形，简单调度架构如图 14-2 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-29-113406.jpg" alt=""> 图 14-2 调度架构 我们需要做的就是在多台主机上同时运行爬虫任务协同爬取，而协同爬取的前提就是共享爬取队列。这样各台主机就不需要各自维护爬取队列，而是从共享爬取队列存取 Request。但是各台主机还是有各自的 Scheduler 和 Downloader，所以调度和下载功能分别完成。如果不考虑队列存取性能消耗，爬取效率还是会成倍提高。</p>
                  <h3 id="2-维护爬取队列"><a href="#2-维护爬取队列" class="headerlink" title="2. 维护爬取队列"></a>2. 维护爬取队列</h3>
                  <p>那么这个队列用什么维护来好呢？我们首先需要考虑的就是性能问题，什么数据库存取效率高？我们自然想到基于内存存储的 Redis，而且 Redis 还支持多种数据结构，例如列表 List、集合 Set、有序集合 Sorted Set 等等，存取的操作也非常简单，所以在这里我们采用 Redis 来维护爬取队列。 这几种数据结构存储实际各有千秋，分析如下：</p>
                  <ul>
                    <li>列表数据结构有 lpush()、lpop()、rpush()、rpop() 方法，所以我们可以用它来实现一个先进先出式爬取队列，也可以实现一个先进后出栈式爬取队列。</li>
                    <li>集合的元素是无序的且不重复的，这样我们可以非常方便地实现一个随机排序的不重复的爬取队列。</li>
                    <li>有序集合带有分数表示，而 Scrapy 的 Request 也有优先级的控制，所以用有集合我们可以实现一个带优先级调度的队列。</li>
                  </ul>
                  <p>这些不同的队列我们需要根据具体爬虫的需求灵活选择。</p>
                  <h3 id="3-怎样来去重"><a href="#3-怎样来去重" class="headerlink" title="3. 怎样来去重"></a>3. 怎样来去重</h3>
                  <p>Scrapy 有自动去重，它的去重使用了 Python 中的集合。这个集合记录了 Scrapy 中每个 Request 的指纹，这个指纹实际上就是 Request 的散列值。我们可以看看 Scrapy 的源代码，如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import hashlib</span><br><span class="line">def request<span class="constructor">_fingerprint(<span class="params">request</span>, <span class="params">include_headers</span>=None)</span>:</span><br><span class="line">    <span class="keyword">if</span> include_headers:</span><br><span class="line">        include_headers = tuple(<span class="keyword">to</span><span class="constructor">_bytes(<span class="params">h</span>.<span class="params">lower</span>()</span>)</span><br><span class="line">                                 for h <span class="keyword">in</span> sorted(include_headers))</span><br><span class="line">    cache = <span class="module-access"><span class="module"><span class="identifier">_fingerprint_cache</span>.</span></span>setdefault(request, &#123;&#125;)</span><br><span class="line">    <span class="keyword">if</span> include_headers not <span class="keyword">in</span> cache:</span><br><span class="line">        fp = hashlib.sha1<span class="literal">()</span></span><br><span class="line">        fp.update(<span class="keyword">to</span><span class="constructor">_bytes(<span class="params">request</span>.<span class="params">method</span>)</span>)</span><br><span class="line">        fp.update(<span class="keyword">to</span><span class="constructor">_bytes(<span class="params">canonicalize_url</span>(<span class="params">request</span>.<span class="params">url</span>)</span>))</span><br><span class="line">        fp.update(request.body <span class="keyword">or</span> b'')</span><br><span class="line">        <span class="keyword">if</span> include_headers:</span><br><span class="line">            for hdr <span class="keyword">in</span> include_headers:</span><br><span class="line">                <span class="keyword">if</span> hdr <span class="keyword">in</span> request.headers:</span><br><span class="line">                    fp.update(hdr)</span><br><span class="line">                    for v <span class="keyword">in</span> request.headers.getlist(hdr):</span><br><span class="line">                        fp.update(v)</span><br><span class="line">        cache<span class="literal">[<span class="identifier">include_headers</span>]</span> = fp.hexdigest<span class="literal">()</span></span><br><span class="line">    return cache<span class="literal">[<span class="identifier">include_headers</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>request_fingerprint() 就是计算 Request 指纹的方法，其方法内部使用的是 hashlib 的 sha1() 方法。计算的字段包括 Request 的 Method、URL、Body、Headers 这几部分内容，这里只要有一点不同，那么计算的结果就不同。计算得到的结果是加密后的字符串，也就是指纹。每个 Request 都有独有的指纹，指纹就是一个字符串，判定字符串是否重复比判定 Request 对象是否重复容易得多，所以指纹可以作为判定 Request 是否重复的依据。 那么我们如何判定重复呢？Scrapy 是这样实现的，如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">self</span>.fingerprints = set()</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">request_seen</span><span class="params">(<span class="keyword">self</span>, request)</span></span><span class="symbol">:</span></span><br><span class="line">    fp = <span class="keyword">self</span>.request_fingerprint(request)</span><br><span class="line">    <span class="keyword">if</span> fp <span class="keyword">in</span> <span class="keyword">self</span>.<span class="symbol">fingerprints:</span></span><br><span class="line">        <span class="keyword">return</span> True</span><br><span class="line">    <span class="keyword">self</span>.fingerprints.add(fp)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在去重的类 RFPDupeFilter 中，有一个 request_seen() 方法，这个方法有一个参数 request，它的作用就是检测该 Request 对象是否重复。这个方法调用 request_fingerprint() 获取该 Request 的指纹，检测这个指纹是否存在于 fingerprints 变量中，而 fingerprints 是一个集合，集合的元素都是不重复的。如果指纹存在，那么就返回 True，说明该 Request 是重复的，否则这个指纹加入到集合中。如果下次还有相同的 Request 传递过来，指纹也是相同的，那么这时指纹就已经存在于集合中，Request 对象就会直接判定为重复。这样去重的目的就实现了。 Scrapy 的去重过程就是，利用集合元素的不重复特性来实现 Request 的去重。 对于分布式爬虫来说，我们肯定不能再用每个爬虫各自的集合来去重了。因为这样还是每个主机单独维护自己的集合，不能做到共享。多台主机如果生成了相同的 Request，只能各自去重，各个主机之间就无法做到去重了。 那么要实现去重，这个指纹集合也需要是共享的，Redis 正好有集合的存储数据结构，我们可以利用 Redis 的集合作为指纹集合，那么这样去重集合也是利用 Redis 共享的。每台主机新生成 Request 之后，把该 Request 的指纹与集合比对，如果指纹已经存在，说明该 Request 是重复的，否则将 Request 的指纹加入到这个集合中即可。利用同样的原理不同的存储结构我们也实现了分布式 Reqeust 的去重。</p>
                  <h3 id="4-防止中断"><a href="#4-防止中断" class="headerlink" title="4. 防止中断"></a>4. 防止中断</h3>
                  <p>在 Scrapy 中，爬虫运行时的 Request 队列放在内存中。爬虫运行中断后，这个队列的空间就被释放，此队列就被销毁了。所以一旦爬虫运行中断，爬虫再次运行就相当于全新的爬取过程。 要做到中断后继续爬取，我们可以将队列中的 Request 保存起来，下次爬取直接读取保存数据即可获取上次爬取的队列。我们在 Scrapy 中指定一个爬取队列的存储路径即可，这个路径使用 JOB_DIR 变量来标识，我们可以用如下命令来实现：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy crawl spider -s <span class="attribute">JOBDIR</span>=crawls/spider</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>更加详细的使用方法可以参见官方文档，链接为：<a href="https://doc.scrapy.org/en/latest/topics/jobs.html" target="_blank" rel="noopener">https://doc.scrapy.org/en/latest/topics/jobs.html</a>。 在 Scrapy 中，我们实际是把爬取队列保存到本地，第二次爬取直接读取并恢复队列即可。那么在分布式架构中我们还用担心这个问题吗？不需要。因为爬取队列本身就是用数据库保存的，如果爬虫中断了，数据库中的 Request 依然是存在的，下次启动就会接着上次中断的地方继续爬取。 所以，当 Redis 的队列为空时，爬虫会重新爬取；当 Redis 的队列不为空时，爬虫便会接着上次中断之处继续爬取。</p>
                  <h3 id="5-架构实现"><a href="#5-架构实现" class="headerlink" title="5. 架构实现"></a>5. 架构实现</h3>
                  <p>我们接下来就需要在程序中实现这个架构了。首先实现一个共享的爬取队列，还要实现去重的功能。另外，重写一个 Scheduer 的实现，使之可以从共享的爬取队列存取 Request。 幸运的是，已经有人实现了这些逻辑和架构，并发布成叫 Scrapy-Redis 的 Python 包。接下来，我们看看 Scrapy-Redis 的源码实现，以及它的详细工作原理。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-08 09:52:07" itemprop="dateCreated datePublished" datetime="2019-12-08T09:52:07+08:00">2019-12-08</time>
                </span>
                <span id="/8456.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 14.1–分布式爬虫原理" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8453.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8453.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-13-Scrapy-爬取新浪微博"><a href="#13-13-Scrapy-爬取新浪微博" class="headerlink" title="13.13 Scrapy 爬取新浪微博"></a>13.13 Scrapy 爬取新浪微博</h1>
                  <p>前面讲解了 Scrapy 中各个模块基本使用方法以及代理池、Cookies 池。接下来我们以一个反爬比较强的网站新浪微博为例，来实现一下 Scrapy 的大规模爬取。</p>
                  <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                  <p>本次爬取的目标是新浪微博用户的公开基本信息，如用户昵称、头像、用户的关注、粉丝列表以及发布的微博等，这些信息抓取之后保存至 MongoDB。</p>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>请确保前文所讲的代理池、Cookies 池已经实现并可以正常运行，安装 Scrapy、PyMongo 库，如没有安装可以参考前文内容。</p>
                  <h3 id="3-爬取思路"><a href="#3-爬取思路" class="headerlink" title="3. 爬取思路"></a>3. 爬取思路</h3>
                  <p>首先我们要实现用户的大规模爬取。这里采用的爬取方式是，以微博的几个大 V 为起始点，爬取他们各自的粉丝和关注列表，然后获取粉丝和关注列表的粉丝和关注列表，以此类推，这样下去就可以实现递归爬取。如果一个用户与其他用户有社交网络上的关联，那他们的信息就会被爬虫抓取到，这样我们就可以做到对所有用户的爬取。通过这种方式，我们可以得到用户的唯一 ID，再根据 ID 获取每个用户发布的微博即可。</p>
                  <h3 id="4-爬取分析"><a href="#4-爬取分析" class="headerlink" title="4. 爬取分析"></a>4. 爬取分析</h3>
                  <p>这里我们选取的爬取站点是：<a href="https://m.weibo.cn，此站点是微博移动端的站点。打开该站点会跳转到登录页面，这是因为主页做了登录限制。不过我们可以直接打开某个用户详情页面，如图" target="_blank" rel="noopener">https://m.weibo.cn，此站点是微博移动端的站点。打开该站点会跳转到登录页面，这是因为主页做了登录限制。不过我们可以直接打开某个用户详情页面，如图</a> 13-32 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034836.jpg" alt=""> 图 13-32 个人详情页面 我们在页面最上方可以看到她的关注和粉丝数量。我们点击关注，进入到她的关注列表，如图 13-33 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034839.jpg" alt=""> 图 13-33 关注列表 我们打开开发者工具，切换到 XHR 过滤器，一直下拉关注列表，即可看到下方会出现很多 Ajax 请求，这些请求就是获取关注列表的 Ajax 请求，如图 13-34 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034852.png" alt=""> 图 13-34 请求列表 我们打开第一个 Ajax 请求看一下，发现它的链接为：<a href="https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_1916655407&amp;luicode=10000011&amp;lfid=1005051916655407&amp;featurecode=20000320&amp;type=uid&amp;value=1916655407&amp;page=2" target="_blank" rel="noopener">https://m.weibo.cn/api/container/getIndex?containerid=231051<em>-_followers</em>-_1916655407&amp;luicode=10000011&amp;lfid=1005051916655407&amp;featurecode=20000320&amp;type=uid&amp;value=1916655407&amp;page=2</a>，详情如图 13-35 和 13-36 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034901.jpg" alt=""> 图 13-35 请求详情 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034906.jpg" alt=""> 图 13-36 响应结果 请求类型是 GET 类型，返回结果是 JSON 格式，我们将其展开之后即可看到其关注的用户的基本信息。接下来我们只需要构造这个请求的参数。此链接一共有 7 个参数，如图 13-37 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034931.jpg" alt=""> 图 13-37 参数信息 其中最主要的参数就是 containerid 和 page。有了这两个参数，我们同样可以获取请求结果。我们可以将接口精简为：<a href="https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_1916655407&amp;page=2" target="_blank" rel="noopener">https://m.weibo.cn/api/container/getIndex?containerid=231051<em>-_followers</em>-_1916655407&amp;page=2</a>，这里的 containerid 的前半部分是固定的，后半部分是用户的 id。所以这里参数就可以构造出来了，只需要修改 containerid 最后的 id 和 page 参数即可获取分页形式的关注列表信息。 利用同样的方法，我们也可以分析用户详情的 Ajax 链接、用户微博列表的 Ajax 链接，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># 用户详情 API</span></span><br><span class="line"><span class="attr">user_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;value=&#123;uid&#125;&amp;containerid=100505&#123;uid&#125;'</span></span><br><span class="line"><span class="comment"># 关注列表 API</span></span><br><span class="line"><span class="attr">follow_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line"><span class="comment"># 粉丝列表 API</span></span><br><span class="line"><span class="attr">fan_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line"><span class="comment"># 微博列表 API</span></span><br><span class="line"><span class="attr">weibo_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;page=&#123;page&#125;&amp;containerid=107603&#123;uid&#125;'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此处的 uid 和 page 分别代表用户 ID 和分页页码。 注意，这个 API 可能随着时间的变化或者微博的改版而变化，以实测为准。 我们从几个大 V 开始抓取，抓取他们的粉丝、关注列表、微博信息，然后递归抓取他们的粉丝和关注列表的粉丝、关注列表、微博信息，递归抓取，最后保存微博用户的基本信息、关注和粉丝列表、发布的微博。 我们选择 MongoDB 作为存储的数据库，可以更方便地存储用户的粉丝和关注列表。</p>
                  <h3 id="5-新建项目"><a href="#5-新建项目" class="headerlink" title="5. 新建项目"></a>5. 新建项目</h3>
                  <p>接下来，我们用 Scrapy 来实现这个抓取过程。首先创建一个项目，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject weibo</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>进入项目中，新建一个 Spider，名为 weibocn，命令如下所示：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">weibocn</span> <span class="selector-tag">m</span><span class="selector-class">.weibo</span><span class="selector-class">.cn</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们首先修改 Spider，配置各个 Ajax 的 URL，选取几个大 V，将他们的 ID 赋值成一个列表，实现 start_requests() 方法，也就是依次抓取各个大 V 的个人详情，然后用 parse_user() 进行解析，如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy import Request, Spider</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">WeiboSpider</span>(<span class="title">Spider</span>):</span></span><br><span class="line">    name = <span class="string">'weibocn'</span></span><br><span class="line">    allowed_domains = [<span class="string">'m.weibo.cn'</span>]</span><br><span class="line">    user_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;value=&#123;uid&#125;&amp;containerid=100505&#123;uid&#125;'</span></span><br><span class="line">    follow_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line">    fan_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line">    weibo_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;page=&#123;page&#125;&amp;containerid=107603&#123;uid&#125;'</span></span><br><span class="line">    start_users = [<span class="string">'3217179555'</span>, <span class="string">'1742566624'</span>, <span class="string">'2282991915'</span>, <span class="string">'1288739185'</span>, <span class="string">'3952070245'</span>, <span class="string">'5878659096'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">for</span> uid <span class="keyword">in</span> <span class="keyword">self</span>.<span class="symbol">start_users:</span></span><br><span class="line">            <span class="keyword">yield</span> Request(<span class="keyword">self</span>.user_url.format(uid=uid), callback=<span class="keyword">self</span>.parse_user)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_user</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.logger.debug(response)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="6-创建-Item"><a href="#6-创建-Item" class="headerlink" title="6. 创建 Item"></a>6. 创建 Item</h3>
                  <p>接下来，我们解析用户的基本信息并生成 Item。这里我们先定义几个 Item，如用户、用户关系、微博的 Item，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Item, Field</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="symbol">UserItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">users</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">name</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">avatar</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">cover</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">gender</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">description</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">weibos_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified_reason</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified_type</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">crawled_at</span> = <span class="symbol">Field</span>()</span><br><span class="line"></span><br><span class="line"><span class="symbol">class</span> <span class="symbol">UserRelationItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">users</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans</span> = <span class="symbol">Field</span>()</span><br><span class="line"></span><br><span class="line"><span class="symbol">class</span> <span class="symbol">WeiboItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">weibos</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">attitudes_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">comments_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">reposts_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">picture</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">pictures</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">source</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">text</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">raw_text</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">thumbnail</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">user</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">created_at</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">crawled_at</span> = <span class="symbol">Field</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了 collection 字段，指明保存的 Collection 的名称。用户的关注和粉丝列表直接定义为一个单独的 UserRelationItem，其中 id 就是用户的 ID，follows 就是用户关注列表，fans 是粉丝列表，但这并不意味着我们会将关注和粉丝列表存到一个单独的 Collection 里。后面我们会用 Pipeline 对各个 Item 进行处理、合并存储到用户的 Collection 里，因此 Item 和 Collection 并不一定是完全对应的。</p>
                  <h3 id="7-提取数据"><a href="#7-提取数据" class="headerlink" title="7. 提取数据"></a>7. 提取数据</h3>
                  <p>我们开始解析用户的基本信息，实现 parse_user() 方法，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_user(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析用户信息</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'userInfo'</span>):</span><br><span class="line">        user_info = result.<span class="builtin-name">get</span>(<span class="string">'userInfo'</span>)</span><br><span class="line">        user_item = UserItem()</span><br><span class="line">        field_map = &#123;</span><br><span class="line">            <span class="string">'id'</span>: <span class="string">'id'</span>, <span class="string">'name'</span>: <span class="string">'screen_name'</span>, <span class="string">'avatar'</span>: <span class="string">'profile_image_url'</span>, <span class="string">'cover'</span>: <span class="string">'cover_image_phone'</span>,</span><br><span class="line">            <span class="string">'gender'</span>: <span class="string">'gender'</span>, <span class="string">'description'</span>: <span class="string">'description'</span>, <span class="string">'fans_count'</span>: <span class="string">'followers_count'</span>,</span><br><span class="line">            <span class="string">'follows_count'</span>: <span class="string">'follow_count'</span>, <span class="string">'weibos_count'</span>: <span class="string">'statuses_count'</span>, <span class="string">'verified'</span>: <span class="string">'verified'</span>,</span><br><span class="line">            <span class="string">'verified_reason'</span>: <span class="string">'verified_reason'</span>, <span class="string">'verified_type'</span>: <span class="string">'verified_type'</span></span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">for</span> field, attr <span class="keyword">in</span> field_map.items():</span><br><span class="line">            user_item[field] = user_info.<span class="builtin-name">get</span>(attr)</span><br><span class="line">        yield user_item</span><br><span class="line">        # 关注</span><br><span class="line">        uid = user_info.<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        yield Request(self.follow_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_follows,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br><span class="line">        # 粉丝</span><br><span class="line">        yield Request(self.fan_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_fans,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br><span class="line">        # 微博</span><br><span class="line">        yield Request(self.weibo_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_weibos,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们一共完成了两个操作。</p>
                  <ul>
                    <li>解析 JSON 提取用户信息并生成 UserItem 返回。我们并没有采用常规的逐个赋值的方法，而是定义了一个字段映射关系。我们定义的字段名称可能和 JSON 中用户的字段名称不同，所以在这里定义成一个字典，然后遍历字典的每个字段实现逐个字段的赋值。</li>
                    <li>构造用户的关注、粉丝、微博的第一页的链接，并生成 Request，这里需要的参数只有用户的 ID。另外，初始分页页码直接设置为 1 即可。</li>
                  </ul>
                  <p>接下来，我们还需要保存用户的关注和粉丝列表。以关注列表为例，其解析方法为 parse_follows()，实现如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_follows(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析用户关注</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'ok'</span>) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>) <span class="keyword">and</span> len(result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)[-1].<span class="builtin-name">get</span>(<span class="string">'card_group'</span>):</span><br><span class="line">        # 解析用户</span><br><span class="line">        follows = result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)[-1].<span class="builtin-name">get</span>(<span class="string">'card_group'</span>)</span><br><span class="line">        <span class="keyword">for</span> follow <span class="keyword">in</span> follows:</span><br><span class="line">            <span class="keyword">if</span> follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>):</span><br><span class="line">                uid = follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">                yield Request(self.user_url.format(<span class="attribute">uid</span>=uid), <span class="attribute">callback</span>=self.parse_user)</span><br><span class="line">        # 关注列表</span><br><span class="line">        uid = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">        user_relation_item = UserRelationItem()</span><br><span class="line">        follows = [&#123;<span class="string">'id'</span>: follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>), <span class="string">'name'</span>: follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'screen_name'</span>)&#125; <span class="keyword">for</span> follow <span class="keyword">in</span></span><br><span class="line">                   follows]</span><br><span class="line">        user_relation_item[<span class="string">'id'</span>] = uid</span><br><span class="line">        user_relation_item[<span class="string">'follows'</span>] = follows</span><br><span class="line">        user_relation_item[<span class="string">'fans'</span>] = []</span><br><span class="line">        yield user_relation_item</span><br><span class="line">        # 下一页关注</span><br><span class="line">       <span class="built_in"> page </span>= response.meta.<span class="builtin-name">get</span>(<span class="string">'page'</span>) + 1</span><br><span class="line">        yield Request(self.follow_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=page),</span><br><span class="line">                      <span class="attribute">callback</span>=self.parse_follows, meta=&#123;<span class="string">'page'</span>: page, <span class="string">'uid'</span>: uid&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>那么在这个方法里面我们做了如下三件事。</p>
                  <ul>
                    <li>解析关注列表中的每个用户信息并发起新的解析请求。我们首先解析关注列表的信息，得到用户的 ID，然后再利用 user_url 构造访问用户详情的 Request，回调就是刚才所定义的 parse_user() 方法。</li>
                    <li>提取用户关注列表内的关键信息并生成 UserRelationItem。id 字段直接设置成用户的 ID，JSON 返回数据中的用户信息有很多冗余字段。在这里我们只提取了关注用户的 ID 和用户名，然后把它们赋值给 follows 字段，fans 字段设置成空列表。这样我们就建立了一个存有用户 ID 和用户部分关注列表的 UserRelationItem，之后合并且保存具有同一个 ID 的 UserRelationItem 的关注和粉丝列表。</li>
                    <li>提取下一页关注。只需要将此请求的分页页码加 1 即可。分页页码通过 Request 的 meta 属性进行传递，Response 的 meta 来接收。这样我们构造并返回下一页的关注列表的 Request。</li>
                  </ul>
                  <p>抓取粉丝列表的原理和抓取关注列表原理相同，在此不再赘述。 接下来我们还差一个方法的实现，即 parse_weibos()，它用来抓取用户的微博信息，实现如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_weibos(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析微博列表</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'ok'</span>) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>):</span><br><span class="line">        weibos = result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)</span><br><span class="line">        <span class="keyword">for</span> weibo <span class="keyword">in</span> weibos:</span><br><span class="line">            mblog = weibo.<span class="builtin-name">get</span>(<span class="string">'mblog'</span>)</span><br><span class="line">            <span class="keyword">if</span> mblog:</span><br><span class="line">                weibo_item = WeiboItem()</span><br><span class="line">                field_map = &#123;</span><br><span class="line">                    <span class="string">'id'</span>: <span class="string">'id'</span>, <span class="string">'attitudes_count'</span>: <span class="string">'attitudes_count'</span>, <span class="string">'comments_count'</span>: <span class="string">'comments_count'</span>, <span class="string">'created_at'</span>: <span class="string">'created_at'</span>,</span><br><span class="line">                    <span class="string">'reposts_count'</span>: <span class="string">'reposts_count'</span>, <span class="string">'picture'</span>: <span class="string">'original_pic'</span>, <span class="string">'pictures'</span>: <span class="string">'pics'</span>,</span><br><span class="line">                    <span class="string">'source'</span>: <span class="string">'source'</span>, <span class="string">'text'</span>: <span class="string">'text'</span>, <span class="string">'raw_text'</span>: <span class="string">'raw_text'</span>, <span class="string">'thumbnail'</span>: <span class="string">'thumbnail_pic'</span></span><br><span class="line">                &#125;</span><br><span class="line">                <span class="keyword">for</span> field, attr <span class="keyword">in</span> field_map.items():</span><br><span class="line">                    weibo_item[field] = mblog.<span class="builtin-name">get</span>(attr)</span><br><span class="line">                weibo_item[<span class="string">'user'</span>] = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">                yield weibo_item</span><br><span class="line">        # 下一页微博</span><br><span class="line">        uid = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">       <span class="built_in"> page </span>= response.meta.<span class="builtin-name">get</span>(<span class="string">'page'</span>) + 1</span><br><span class="line">        yield Request(self.weibo_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=page), <span class="attribute">callback</span>=self.parse_weibos,</span><br><span class="line">                      meta=&#123;<span class="string">'uid'</span>: uid, <span class="string">'page'</span>: page&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 parse_weibos() 方法完成了两件事。</p>
                  <ul>
                    <li>提取用户的微博信息，并生成 WeiboItem。这里同样建立了一个字段映射表，实现批量字段赋值。</li>
                    <li>提取下一页的微博列表。这里同样需要传入用户 ID 和分页页码。</li>
                  </ul>
                  <p>到目前为止，微博的 Spider 已经完成。后面还需要对数据进行数据清洗存储，以及对接代理池、Cookies 池来防止反爬虫。</p>
                  <h3 id="8-数据清洗"><a href="#8-数据清洗" class="headerlink" title="8. 数据清洗"></a>8. 数据清洗</h3>
                  <p>有些微博的时间可能不是标准的时间，比如它可能显示为刚刚、几分钟前、几小时前、昨天等。这里我们需要统一转化这些时间，实现一个 parse_time() 方法，如下所示：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_time(self, <span class="type">date</span>):</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">' 刚刚 '</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time()))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d + 分钟前 '</span>, <span class="type">date</span>):</span><br><span class="line">        minute = re.match(<span class="string">'(d+)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>)</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time() - <span class="type">float</span>(minute) * <span class="number">60</span>))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d + 小时前 '</span>, <span class="type">date</span>):</span><br><span class="line">        hour = re.match(<span class="string">'(d+)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>)</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time() - <span class="type">float</span>(hour) * <span class="number">60</span> * <span class="number">60</span>))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">' 昨天.*'</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = re.match(<span class="string">' 昨天 (.*)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>).strip()</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>() - <span class="number">24</span> * <span class="number">60</span> * <span class="number">60</span>) + <span class="string">' '</span> + <span class="type">date</span></span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d&#123;2&#125;-d&#123;2&#125;'</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>()) + <span class="type">date</span> + <span class="string">' 00:00'</span></span><br><span class="line">    <span class="keyword">return</span> <span class="type">date</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们用正则来提取一些关键数字，用 time 库来实现标准时间的转换。 以 X 分钟前的处理为例，爬取的时间会赋值为 created_at 字段。我们首先用正则匹配这个时间，表达式写作 d + 分钟前，如果提取到的时间符合这个表达式，那么就提取出其中的数字，这样就可以获取分钟数了。接下来使用 time 模块的 strftime() 方法，第一个参数传入要转换的时间格式，第二个参数就是时间戳。这里我们用当前的时间戳减去此分钟数乘以 60 就是当时的时间戳，这样我们就可以得到格式化后的正确时间了。 然后 Pipeline 可以实现如下处理：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">class WeiboPipeline():</span><br><span class="line">    def process_item(self, <span class="keyword">item</span>, spider):</span><br><span class="line">        <span class="keyword">if</span> isinstance(<span class="keyword">item</span>, WeiboItem):</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'created_at'</span>):</span><br><span class="line">                <span class="keyword">item</span>[<span class="string">'created_at'</span>] = <span class="keyword">item</span>[<span class="string">'created_at'</span>].strip()</span><br><span class="line">                <span class="keyword">item</span>[<span class="string">'created_at'</span>] = self.parse_time(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'created_at'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们在 Spider 里没有对 crawled_at 字段赋值，它代表爬取时间，我们可以统一将其赋值为当前时间，实现如下所示：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">TimePipeline</span>():</span></span><br><span class="line"><span class="class">    def process_item(<span class="title">self</span>, <span class="title">item</span>, <span class="title">spider</span>):</span></span><br><span class="line"><span class="class">        if isinstance(<span class="title">item</span>, <span class="type">UserItem</span>) or isinstance(<span class="title">item</span>, <span class="type">WeiboItem</span>):</span></span><br><span class="line"><span class="class">            now = time.strftime('% <span class="type">Y</span>-% <span class="title">m</span>-% <span class="title">d</span> % <span class="type">H</span>:% <span class="type">M</span>', <span class="title">time</span>.<span class="title">localtime</span>())</span></span><br><span class="line"><span class="class">            item['crawled_at'] = now</span></span><br><span class="line"><span class="class">        return item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们判断了 item 如果是 UserItem 或 WeiboItem 类型，那么就给它的 crawled_at 字段赋值为当前时间。 通过上面的两个 Pipeline，我们便完成了数据清洗工作，这里主要是时间的转换。</p>
                  <h3 id="9-数据存储"><a href="#9-数据存储" class="headerlink" title="9. 数据存储"></a>9. 数据存储</h3>
                  <p>数据清洗完毕之后，我们就要将数据保存到 MongoDB 数据库。我们在这里实现 MongoPipeline 类，如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymongo</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>), mongo_db=crawler.settings.get(<span class="string">'MONGO_DATABASE'</span>)</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line">        <span class="keyword">self</span>.db[UserItem.collection].create_index([(<span class="string">'id'</span>, pymongo.ASCENDING)])</span><br><span class="line">        <span class="keyword">self</span>.db[WeiboItem.collection].create_index([(<span class="string">'id'</span>, pymongo.ASCENDING)])</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> isinstance(item, UserItem) <span class="keyword">or</span> isinstance(item, WeiboItem)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.db[item.collection].update(&#123;<span class="string">'id'</span>: item.get(<span class="string">'id'</span>)&#125;, &#123;<span class="string">'$set'</span>: item&#125;, True)</span><br><span class="line">        <span class="keyword">if</span> isinstance(item, UserRelationItem)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.db[item.collection].update(&#123;<span class="string">'id'</span>: item.get(<span class="string">'id'</span>)&#125;,</span><br><span class="line">                &#123;<span class="string">'$addToSet'</span><span class="symbol">:</span></span><br><span class="line">                    &#123;<span class="string">'follows'</span>: &#123;<span class="string">'$each'</span>: item[<span class="string">'follows'</span>]&#125;,</span><br><span class="line">                        <span class="string">'fans'</span>: &#123;<span class="string">'$each'</span>: item[<span class="string">'fans'</span>]&#125;</span><br><span class="line">                    &#125;</span><br><span class="line">                &#125;, True)</span><br><span class="line">        <span class="keyword">return</span> item</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当前的 MongoPipeline 和前面我们所写的有所不同，主要有以下几点。</p>
                  <ul>
                    <li>在 open_spider() 方法里面添加了 Collection 的索引，在这里为两个 Item 都做了索引，索引的字段是 id，由于我们这次是大规模爬取，同时在爬取过程中涉及到数据的更新问题，所以我们为每个 Collection 建立了索引，建立了索引之后可以大大提高检索效率。</li>
                    <li>在 process_item() 方法里存储使用的是 update() 方法，第一个参数是查询条件，第二个参数是爬取的 Item，这里我们使用了 $set 操作符，这样我们如果爬取到了重复的数据即可对数据进行更新，同时不会删除已存在的字段，如果这里不加 $set 操作符，那么会直接进行 item 替换，这样可能会导致已存在的字段如关注和粉丝列表清空，所以这里必须要加上 $set 操作符。第三个参数我们设置为了 True，这个参数起到的作用是如果数据不存在，则插入数据。这样我们就可以做到数据存在即更新、数据不存在即插入，这样就达到了去重的效果。</li>
                    <li>对于用户的关注和粉丝列表，我们在这里使用了一个新的操作符，叫做 $addToSet，这个操作符可以向列表类型的字段插入数据同时去重，接下来它的值就是需要操作的字段名称，我们在这里又利用了 $each 操作符对需要插入的列表数据进行了遍历，这样就可以逐条插入用户的关注或粉丝数据到指定的字段了，关于该操作更多的解释可以参考 MongoDB 的官方文档，链接为：<a href="https://docs.mongodb.com/manual/reference/operator/update/addToSet/" target="_blank" rel="noopener">https://docs.mongodb.com/manual/reference/operator/update/addToSet/</a>。</li>
                  </ul>
                  <h3 id="10-Cookies-池对接"><a href="#10-Cookies-池对接" class="headerlink" title="10. Cookies 池对接"></a>10. Cookies 池对接</h3>
                  <p>新浪微博的反爬能力非常强，我们需要做一些防范反爬虫的措施才可以顺利完成数据爬取。 如果没有登录而直接请求微博的 API 接口，这非常容易导致 403 状态码。这个情况我们在 10.2 节也提过。所以在这里我们实现一个 Middleware，为每个 Request 添加随机的 Cookies。 我们先开启 Cookies 池，使 API 模块正常运行。例如在本地运行 5000 端口，访问：<a href="http://localhost:5000/weibo/random" target="_blank" rel="noopener">http://localhost:5000/weibo/random</a> 即可获取随机的 Cookies，当然也可以将 Cookies 池部署到远程的服务器，这样只需要更改一下访问的链接就好了。 那么在这里我们将 Cookies 池在本地启动起来，再实现一个 Middleware 如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">CookiesMiddleware</span>():</span></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, cookies_url)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="keyword">self</span>.logger = logging.getLogger(__name_<span class="number">_</span>)</span><br><span class="line">       <span class="keyword">self</span>.cookies_url = cookies_url</span><br><span class="line"></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">get_random_cookies</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="symbol">try:</span></span><br><span class="line">           response = requests.get(<span class="keyword">self</span>.cookies_url)</span><br><span class="line">           <span class="keyword">if</span> response.status_code == <span class="number">200</span><span class="symbol">:</span></span><br><span class="line">               cookies = json.loads(response.text)</span><br><span class="line">               <span class="keyword">return</span> cookies</span><br><span class="line">       except requests.<span class="symbol">ConnectionError:</span></span><br><span class="line">           <span class="keyword">return</span> False</span><br><span class="line"></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">process_request</span><span class="params">(<span class="keyword">self</span>, request, spider)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="keyword">self</span>.logger.debug(<span class="string">' 正在获取 Cookies'</span>)</span><br><span class="line">       cookies = <span class="keyword">self</span>.get_random_cookies()</span><br><span class="line">       <span class="keyword">if</span> <span class="symbol">cookies:</span></span><br><span class="line">           request.cookies = cookies</span><br><span class="line">           <span class="keyword">self</span>.logger.debug(<span class="string">' 使用 Cookies '</span> + json.dumps(cookies))</span><br><span class="line"></span><br><span class="line">   @classmethod</span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">       settings = crawler.settings</span><br><span class="line">       <span class="keyword">return</span> cls(cookies_url=settings.get(<span class="string">'COOKIES_URL'</span>)</span><br><span class="line">       )</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们首先利用 from_crawler() 方法获取了 COOKIES_URL 变量，它定义在 settings.py 里，这就是刚才我们所说的接口。接下来实现 get_random_cookies() 方法，这个方法主要就是请求此 Cookies 池接口并获取接口返回的随机 Cookies。如果成功获取，则返回 Cookies；否则返回 False。 接下来，在 process_request() 方法里，我们给 request 对象的 cookies 属性赋值，其值就是获取的随机 Cookies，这样我们就成功地为每一次请求赋值 Cookies 了。 如果启用了该 Middleware，每个请求都会被赋值随机的 Cookies。这样我们就可以模拟登录之后的请求，403 状态码基本就不会出现。</p>
                  <h3 id="11-代理池对接"><a href="#11-代理池对接" class="headerlink" title="11. 代理池对接"></a>11. 代理池对接</h3>
                  <p>微博还有一个反爬措施就是，检测到同一 IP 请求量过大时就会出现 414 状态码。如果遇到这样的情况可以切换代理。例如，在本地 5555 端口运行，获取随机可用代理的地址为：<a href="http://localhost:5555/random" target="_blank" rel="noopener">http://localhost:5555/random</a>，访问这个接口即可获取一个随机可用代理。接下来我们再实现一个 Middleware，代码如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">class ProxyMiddleware():</span><br><span class="line">    def __init__(self, proxy_url):</span><br><span class="line">        self.logger = logging.getLogger(__name__)</span><br><span class="line">        self.proxy_url = proxy_url</span><br><span class="line"></span><br><span class="line">    def get_random_proxy(self):</span><br><span class="line">        try:</span><br><span class="line">            response = requests.<span class="builtin-name">get</span>(self.proxy_url)</span><br><span class="line">            <span class="keyword">if</span> response.status_code == 200:</span><br><span class="line">               <span class="built_in"> proxy </span>= response.text</span><br><span class="line">                return proxy</span><br><span class="line">        except requests.ConnectionError:</span><br><span class="line">            return <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">    def process_request(self, request, spider):</span><br><span class="line">        <span class="keyword">if</span> request.meta.<span class="builtin-name">get</span>(<span class="string">'retry_times'</span>):</span><br><span class="line">           <span class="built_in"> proxy </span>= self.get_random_proxy()</span><br><span class="line">            <span class="keyword">if</span> proxy:</span><br><span class="line">                uri = <span class="string">'https://&#123;proxy&#125;'</span>.format(<span class="attribute">proxy</span>=proxy)</span><br><span class="line">                self.logger.<span class="builtin-name">debug</span>(<span class="string">' 使用代理 '</span> + proxy)</span><br><span class="line">                request.meta[<span class="string">'proxy'</span>] = uri</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def from_crawler(cls, crawler):</span><br><span class="line">       <span class="built_in"> settings </span>= crawler.settings</span><br><span class="line">        return cls(<span class="attribute">proxy_url</span>=settings.get('PROXY_URL')</span><br><span class="line">        )</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样的原理，我们实现了一个 get_random_proxy() 方法用于请求代理池的接口获取随机代理。如果获取成功，则返回改代理，否则返回 False。在 process_request() 方法中，我们给 request 对象的 meta 属性赋值一个 proxy 字段，该字段的值就是代理。 另外，赋值代理的判断条件是当前 retry_times 不为空，也就是说第一次请求失败之后才启用代理，因为使用代理后访问速度会慢一些。所以我们在这里设置了只有重试的时候才启用代理，否则直接请求。这样就可以保证在没有被封禁的情况下直接爬取，保证了爬取速度。</p>
                  <h3 id="12-启用-Middleware"><a href="#12-启用-Middleware" class="headerlink" title="12. 启用 Middleware"></a>12. 启用 Middleware</h3>
                  <p>接下来，我们在配置文件中启用这两个 Middleware，修改 settings.py 如下所示：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">DOWNLOADER_MIDDLEWARES</span> <span class="string">=</span> <span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'weibo.middlewares.CookiesMiddleware':</span> <span class="number">554</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'weibo.middlewares.ProxyMiddleware':</span> <span class="number">555</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>注意这里的优先级设置，前文提到了 Scrapy 的默认 Downloader Middleware 的设置如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware':</span> <span class="number">100</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware':</span> <span class="number">300</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':</span> <span class="number">350</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':</span> <span class="number">400</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':</span> <span class="number">500</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.retry.RetryMiddleware':</span> <span class="number">550</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware':</span> <span class="number">560</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware':</span> <span class="number">580</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':</span> <span class="number">590</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.RedirectMiddleware':</span> <span class="number">600</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':</span> <span class="number">700</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':</span> <span class="number">750</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.stats.DownloaderStats':</span> <span class="number">850</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware':</span> <span class="number">900</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>要使得我们自定义的 CookiesMiddleware 生效，它在内置的 CookiesMiddleware 之前调用。内置的 CookiesMiddleware 的优先级为 700，所以这里我们设置一个比 700 小的数字即可。 要使得我们自定义的 ProxyMiddleware 生效，它在内置的 HttpProxyMiddleware 之前调用。内置的 HttpProxyMiddleware 的优先级为 750，所以这里我们设置一个比 750 小的数字即可。</p>
                  <h3 id="13-运行"><a href="#13-运行" class="headerlink" title="13. 运行"></a>13. 运行</h3>
                  <p>到此为止，整个微博爬虫就实现完毕了，我们运行如下命令启动一下爬虫：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl weibocn</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>类似的输出结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[urllib3.connectionpool]</span> <span class="attr">DEBUG:</span> <span class="string">http://localhost:5000</span> <span class="string">"GET /weibo/random HTTP/1.1"</span> <span class="number">200</span> <span class="number">339</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[weibo.middlewares]</span> <span class="attr">DEBUG:</span> <span class="string">使用</span> <span class="string">Cookies</span> <span class="string">&#123;"SCF":</span> <span class="string">"AhzwTr_DxIGjgri_dt46_DoPzUqq-PSupu545JdozdHYJ7HyEb4pD3pe05VpbIpVyY1ciKRRWwUgojiO3jYwlBE."</span><span class="string">,</span> <span class="attr">"_T_WM":</span> <span class="string">"8fe0bc1dad068d09b888d8177f1c1218"</span><span class="string">,</span> <span class="attr">"SSOLoginState":</span> <span class="string">"1501496388"</span><span class="string">,</span> <span class="attr">"M_WEIBOCN_PARAMS":</span> <span class="string">"uicode%3D20000174"</span><span class="string">,</span> <span class="attr">"SUHB":</span> <span class="string">"0tKqV4asxqYl4J"</span><span class="string">,</span> <span class="attr">"SUB":</span> <span class="string">"_2A250e3QUDeRhGeBM6VYX8y7NwjiIHXVXhBxcrDV6PUJbkdBeLXjckW2fUT8MWloekO4FCWVlIYJGJdGLnA.."</span><span class="string">&#125;</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[weibocn]</span> <span class="attr">DEBUG:</span> <span class="string">&lt;200</span> <span class="string">https://m.weibo.cn/api/container/getIndex?uid=1742566624&amp;type=uid&amp;value=1742566624&amp;containerid=1005051742566624&gt;</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[scrapy.core.scraper]</span> <span class="attr">DEBUG:</span> <span class="string">Scraped</span> <span class="string">from</span> <span class="string">&lt;200</span> <span class="string">https://m.weibo.cn/api/container/getIndex?uid=1742566624&amp;type=uid&amp;value=1742566624&amp;containerid=1005051742566624&gt;</span></span><br><span class="line"><span class="string">&#123;'avatar':</span> <span class="string">'https://tva4.sinaimg.cn/crop.0.0.180.180.180/67dd74e0jw1e8qgp5bmzyj2050050aa8.jpg'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'cover':</span> <span class="string">'https://tva3.sinaimg.cn/crop.0.0.640.640.640/6ce2240djw1e9oaqhwllzj20hs0hsdir.jpg'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'crawled_at':</span> <span class="string">'2017-07-11 17:27'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'description':</span> <span class="string">' 成长，就是一个不断觉得以前的自己是个傻逼的过程 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'fans_count':</span> <span class="number">19202906</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'follows_count':</span> <span class="number">1599</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'gender':</span> <span class="string">'m'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'id':</span> <span class="number">1742566624</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'name':</span> <span class="string">' 思想聚焦 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified':</span> <span class="literal">True</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified_reason':</span> <span class="string">' 微博知名博主，校导网编辑 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified_type':</span> <span class="number">0</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'weibos_count':</span> <span class="number">58393</span><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行一段时间后，我们便可以到 MongoDB 数据库查看数据，爬取下来的数据如图 13-38 和图 13-39 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034947.jpg" alt=""> 图 13-38 用户信息 <img src="https://qiniu.cuiqingcai.com/2019-11-27-035001.jpg" alt=""> 图 13-39 微博信息 针对用户信息，我们不仅爬取了其基本信息，还把关注和粉丝列表加到了 follows 和 fans 字段并做了去重操作。针对微博信息，我们成功进行了时间转换处理，同时还保存了微博的图片列表信息。</p>
                  <h3 id="14-本节代码"><a href="#14-本节代码" class="headerlink" title="14. 本节代码"></a>14. 本节代码</h3>
                  <p>本节代码地址：<a href="https://github.com/Python3WebSpider/Weibo" target="_blank" rel="noopener">https://github.com/Python3WebSpider/Weibo</a>。</p>
                  <h3 id="15-结语"><a href="#15-结语" class="headerlink" title="15. 结语"></a>15. 结语</h3>
                  <p>本节实现了新浪微博的用户及其粉丝关注列表和微博信息的爬取，还对接了 Cookies 池和代理池来处理反爬虫。不过现在是针对单机的爬取，后面我们会将此项目修改为分布式爬虫，以进一步提高抓取效率。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-08 09:50:48" itemprop="dateCreated datePublished" datetime="2019-12-08T09:50:48+08:00">2019-12-08</time>
                </span>
                <span id="/8453.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>17k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>15 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8448.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8448.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.12–Scrapy 对接 Docker</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-12-Scrapy-对接-Docker"><a href="#13-12-Scrapy-对接-Docker" class="headerlink" title="13.12 Scrapy 对接 Docker"></a>13.12 Scrapy 对接 Docker</h1>
                  <p>环境配置问题可能一直是我们头疼的，我们可能遇到过如下的情况：</p>
                  <ul>
                    <li>我们在本地写好了一个 Scrapy 爬虫项目，想要把它放到服务器上运行，但是服务器上没有安装 Python 环境。</li>
                    <li>别人给了我们一个 Scrapy 爬虫项目，项目中使用包的版本和我们本地环境版本不一致，无法直接运行。</li>
                    <li>我们需要同时管理不同版本的 Scrapy 项目，如早期的项目依赖于 Scrapy 0.25，现在的项目依赖于 Scrapy 1.4.0。</li>
                  </ul>
                  <p>在这些情况下，我们需要解决的就是环境的安装配置、环境的版本冲突解决等问题。 对于 Python 来说，VirtualEnv 的确可以解决版本冲突的问题。但是，VirtualEnv 不太方便做项目部署，我们还是需要安装 Python 环境， 如何解决上述问题呢？答案是用 Docker。Docker 可以提供操作系统级别的虚拟环境，一个 Docker 镜像一般都包含一个完整的操作系统，而这些系统内也有已经配置好的开发环境，如 Python 3.6 环境等。 我们可以直接使用此 Docker 的 Python 3 镜像运行一个容器，将项目直接放到容器里运行，就不用再额外配置 Python 3 环境。这样就解决了环境配置的问题。 我们也可以进一步将 Scrapy 项目制作成一个新的 Docker 镜像，镜像里只包含适用于本项目的 Python 环境。如果要部署到其他平台，只需要下载该镜像并运行就好了，因为 Docker 运行时采用虚拟环境，和宿主机是完全隔离的，所以也不需要担心环境冲突问题。 如果我们能够把 Scrapy 项目制作成一个 Docker 镜像，只要其他主机安装了 Docker，那么只要将镜像下载并运行即可，而不必再担心环境配置问题或版本冲突问题。 接下来，我们尝试把一个 Scrapy 项目制作成一个 Docker 镜像。</p>
                  <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                  <p>我们要实现把前文 Scrapy 的入门项目打包成一个 Docker 镜像的过程。项目爬取的网址为：<a href="http://quotes.toscrape.com/" target="_blank" rel="noopener">http://quotes.toscrape.com/</a>，本章 Scrapy 入门一节已经实现了 Scrapy 对此站点的爬取过程，项目代码为：<a href="https://github.com/Python3WebSpider/ScrapyTutorial" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapyTutorial</a>，如果本地不存在的话可以 Clone 下来。</p>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>请确保已经安装好 Docker 和 MongoDB 并可以正常运行，如果没有安装可以参考第 1 章的安装说明。</p>
                  <h3 id="3-创建-Dockerfile"><a href="#3-创建-Dockerfile" class="headerlink" title="3. 创建 Dockerfile"></a>3. 创建 Dockerfile</h3>
                  <p>首先在项目的根目录下新建一个 requirements.txt 文件，将整个项目依赖的 Python 环境包都列出来，如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy</span></span><br><span class="line"><span class="attribute">pymongo</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果库需要特定的版本，我们还可以指定版本号，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy&gt;=<span class="number">1.4</span><span class="number">.0</span></span><br><span class="line">pymongo&gt;=<span class="number">3.4</span><span class="number">.0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在项目根目录下新建一个 Dockerfile 文件，文件不加任何后缀名，修改内容如下所示：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">FROM</span> python:<span class="number">3.6</span></span><br><span class="line"><span class="keyword">ENV</span> PATH /usr/local/bin:$PATH</span><br><span class="line"><span class="keyword">ADD</span><span class="bash"> . /code</span></span><br><span class="line"><span class="keyword">WORKDIR</span><span class="bash"> /code</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> pip3 install -r requirements.txt</span></span><br><span class="line"><span class="keyword">CMD</span><span class="bash"> scrapy crawl quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一行的 FROM 代表使用的 Docker 基础镜像，在这里我们直接使用 python:3.6 的镜像，在此基础上运行 Scrapy 项目。 第二行 ENV 是环境变量设置，将 /usr/local/bin:$PATH 赋值给 PATH，即增加 /usr/local/bin 这个环境变量路径。 第三行 ADD 是将本地的代码放置到虚拟容器中。它有两个参数：第一个参数是.，代表本地当前路径；第二个参数是 /code，代表虚拟容器中的路径，也就是将本地项目所有内容放置到虚拟容器的 /code 目录下，以便于在虚拟容器中运行代码。 第四行 WORKDIR 是指定工作目录，这里将刚才添加的代码路径设成工作路径。这个路径下的目录结构和当前本地目录结构是相同的，所以我们可以直接执行库安装命令、爬虫运行命令等。 第五行 RUN 是执行某些命令来做一些环境准备工作。由于 Docker 虚拟容器内只有 Python 3 环境，而没有所需要的 Python 库，所以我们运行此命令来在虚拟容器中安装相应的 Python 库如 Scrapy，这样就可以在虚拟容器中执行 Scrapy 命令了。 第六行 CMD 是容器启动命令。在容器运行时，此命令会被执行。在这里我们直接用 scrapy crawl quotes 来启动爬虫。</p>
                  <h3 id="4-修改-MongoDB-连接"><a href="#4-修改-MongoDB-连接" class="headerlink" title="4. 修改 MongoDB 连接"></a>4. 修改 MongoDB 连接</h3>
                  <p>接下来我们需要修改 MongoDB 的连接信息。如果我们继续用 localhost 是无法找到 MongoDB 的，因为在 Docker 虚拟容器里 localhost 实际指向容器本身的运行 IP，而容器内部并没有安装 MongoDB，所以爬虫无法连接 MongoDB。 这里的 MongoDB 地址可以有如下两种选择。</p>
                  <ul>
                    <li>如果只想在本机测试，我们可以将地址修改为宿主机的 IP，也就是容器外部的本机 IP，一般是一个局域网 IP，使用 ifconfig 命令即可查看。</li>
                    <li>如果要部署到远程主机运行，一般 MongoDB 都是可公网访问的地址，修改为此地址即可。</li>
                  </ul>
                  <p>在本节中，我们的目标是将项目打包成一个镜像，让其他远程主机也可运行这个项目。所以我们直接将此处 MongoDB 地址修改为某个公网可访问的远程数据库地址，修改 MONGO_URI 如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MONGO_URI</span> = <span class="string">'mongodb://admin:admin123@120.27.34.25:27017'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此处地址可以修改为自己的远程 MongoDB 数据库地址。 这样项目的配置就完成了。</p>
                  <h3 id="5-构建镜像"><a href="#5-构建镜像" class="headerlink" title="5. 构建镜像"></a>5. 构建镜像</h3>
                  <p>接下来我们便可以构建镜像了，执行如下命令：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">docker</span> <span class="keyword">build </span>-t quotes:latest .</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样的输出就说明镜像构建成功。这时我们查看一下构建的镜像，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Sending build context to Docker daemon <span class="number">191.5</span> kB</span><br><span class="line">Step <span class="number">1</span>/<span class="number">6</span> : FROM python:<span class="number">3.6</span></span><br><span class="line"> ---&gt; <span class="number">968120</span>d8cbe8</span><br><span class="line">Step <span class="number">2</span>/<span class="number">6</span> : ENV PATH /usr/local/bin:$PATH</span><br><span class="line"> ---&gt; Using cache</span><br><span class="line"> ---&gt; <span class="number">387</span>abbba1189</span><br><span class="line">Step <span class="number">3</span>/<span class="number">6</span> : ADD . /code</span><br><span class="line"> ---&gt; a844ee0db9c6</span><br><span class="line">Removing <span class="built_in">int</span>ermediate container <span class="number">4</span>dc41779c573</span><br><span class="line">Step <span class="number">4</span>/<span class="number">6</span> : WORKDIR /code</span><br><span class="line"> ---&gt; <span class="number">619</span>b2c064ae9</span><br><span class="line">Removing <span class="built_in">int</span>ermediate container bcd7cd7f7337</span><br><span class="line">Step <span class="number">5</span>/<span class="number">6</span> : RUN pip3 install -r requirements.txt</span><br><span class="line"> ---&gt; Running <span class="keyword">in</span> <span class="number">9452</span>c83a12c5</span><br><span class="line">...</span><br><span class="line">Removing <span class="built_in">int</span>ermediate container <span class="number">9452</span>c83a12c5</span><br><span class="line">Step <span class="number">6</span>/<span class="number">6</span> : CMD scrapy crawl quotes</span><br><span class="line"> ---&gt; Running <span class="keyword">in</span> c092b5557ab8</span><br><span class="line"> ---&gt; c8101aca6e2a</span><br><span class="line">Removing <span class="built_in">int</span>ermediate container c092b5557ab8</span><br><span class="line">Successfully built c8101aca6e2a</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>出现类似输出就证明镜像构建成功了，这时执行如我们查看一下构建的镜像：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">docker images</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果中其中有一行就是：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">quotes  latest  <span class="number">41</span>c8499ce210    <span class="number">2</span> minutes ago   <span class="number">769</span> MB</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就是我们新构建的镜像。</p>
                  <h3 id="6-运行"><a href="#6-运行" class="headerlink" title="6. 运行"></a>6. 运行</h3>
                  <p>我们可以先在本地测试运行，执行如下命令：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker <span class="keyword">run</span><span class="bash"> quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就利用此镜像新建并运行了一个 Docker 容器，运行效果完全一致，如图 13-29 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034816.jpg" alt=""> 图 13-32 运行结果 如果出现类似图 13-29 的运行结果，这就证明构建的镜像没有问题。</p>
                  <h3 id="7-推送至-Docker-Hub"><a href="#7-推送至-Docker-Hub" class="headerlink" title="7. 推送至 Docker Hub"></a>7. 推送至 Docker Hub</h3>
                  <p>构建完成之后，我们可以将镜像 Push 到 Docker 镜像托管平台，如 Docker Hub 或者私有的 Docker Registry 等，这样我们就可以从远程服务器下拉镜像并运行了。 以 Docker Hub 为例，如果项目包含一些私有的连接信息（如数据库），我们最好将 Repository 设为私有或者直接放到私有的 Docker Registry。 首先在 <a href="https://hub.docker.com" target="_blank" rel="noopener">https://hub.docker.com</a> 注册一个账号，新建一个 Repository，名为 quotes。比如，我的用户名为 germey，新建的 Repository 名为 quotes，那么此 Repository 的地址就可以用 germey/quotes 来表示。 为新建的镜像打一个标签，命令如下所示：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker <span class="keyword">tag</span> <span class="title">quotes</span>:latest germey/quotes:latest</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>推送镜像到 Docker Hub 即可，命令如下所示：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">docker</span> <span class="keyword">push </span>germey/quotes</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>Docker Hub 便会出现新推送的 Docker 镜像了，如图 13-30 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034823.png" alt=""> 图 13-30 推送结果 如果我们想在其他的主机上运行这个镜像，主机上装好 Docker 后，可以直接执行如下命令：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker <span class="keyword">run</span><span class="bash"> germey/quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就会自动下载镜像，然后启动容器运行，不需要配置 Python 环境，不需要关心版本冲突问题。 运行效果如图 13-31 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034829.jpg" alt=""> 图 13-31 运行效果 整个项目爬取完成后，数据就可以存储到指定的数据库中。</p>
                  <h3 id="8-结语"><a href="#8-结语" class="headerlink" title="8. 结语"></a>8. 结语</h3>
                  <p>我们讲解了将 Scrapy 项目制作成 Docker 镜像并部署到远程服务器运行的过程。使用此种方式，我们在本节开头所列出的问题都迎刃而解。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-07 10:23:45" itemprop="dateCreated datePublished" datetime="2019-12-07T10:23:45+08:00">2019-12-07</time>
                </span>
                <span id="/8448.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.12–Scrapy 对接 Docker" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8445.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8445.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.11–Scrapyrt 的使用</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-11-Scrapyrt-的使用"><a href="#13-11-Scrapyrt-的使用" class="headerlink" title="13.11 Scrapyrt 的使用"></a>13.11 Scrapyrt 的使用</h1>
                  <p>Scrapyrt 为 Scrapy 提供了一个调度的 HTTP 接口。有了它我们不需要再执行 Scrapy 命令，而是通过请求一个 HTTP 接口即可调度 Scrapy 任务，我们就不需要借助于命令行来启动项目了。如果项目是在远程服务器运行，利用它来启动项目是个不错的选择。</p>
                  <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                  <p>我们以本章 Scrapy 入门项目为例来说明 Scrapyrt 的使用方法，项目源代码地址为：<a href="https://github.com/Python3WebSpider/ScrapyTutorial" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapyTutorial</a>。</p>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>请确保 Scrapyrt 已经正确安装并正常运行，具体安装可以参考第 1 章的说明。</p>
                  <h3 id="3-启动服务"><a href="#3-启动服务" class="headerlink" title="3. 启动服务"></a>3. 启动服务</h3>
                  <p>首先将项目下载下来，在项目目录下运行 Scrapyrt，假设当前服务运行在 9080 端口上。下面将简单介绍 Scrapyrt 的使用方法。</p>
                  <h3 id="4-GET-请求"><a href="#4-GET-请求" class="headerlink" title="4. GET 请求"></a>4. GET 请求</h3>
                  <p>目前，GET 请求方式支持如下的参数。</p>
                  <ul>
                    <li>spider_name，Spider 名称，字符串类型，必传参数，如果传递的 Spider 名称不存在则会返回 404 错误。</li>
                    <li>url，爬取链接，字符串类型，如果起始链接没有定义的话就必须要传递，如果传递了该参数，Scrapy 会直接用该 URL 生成 Request，而直接忽略 start_requests() 方法和 start_urls 属性的定义。</li>
                    <li>callback，回调函数名称，字符串类型，可选参数，如果传递了就会使用此回调函数处理，否则会默认使用 Spider 内定义的回调函数。</li>
                    <li>max_requests，最大请求数量，数值类型，可选参数，它定义了 Scrapy 执行请求的 Request 的最大限制，如定义为 5，则最多只执行 5 次 Request 请求，其余的则会被忽略。</li>
                    <li>start_requests，是否要执行 start_request() 函数，布尔类型，可选参数，在 Scrapy 项目中如果定义了 start_requests() 方法，那么在项目启动时会默认调用该方法，但是在 Scrapyrt 就不一样了，它默认不执行 start_requests() 方法，如果要执行，需要将它设置为 true。</li>
                  </ul>
                  <p>例如我们执行如下命令：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http:<span class="regexp">//</span>localhost:<span class="number">9080</span><span class="regexp">/crawl.json?spider_name=quotes&amp;url=http:/</span><span class="regexp">/quotes.toscrape.com/</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>得到类似如下结果，如图 13-28 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034631.jpg" alt=""> 图 13-28 输出结果 返回的是一个 JSON 格式的字符串，我们解析它的结构，如下所示：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"status"</span>: <span class="string">"ok"</span>,</span><br><span class="line">  <span class="attr">"items"</span>: [</span><br><span class="line">    &#123;</span><br><span class="line">      <span class="attr">"text"</span>: <span class="string">"“The world as we have created it is a process of o..."</span>,</span><br><span class="line">      <span class="attr">"author"</span>: <span class="string">"Albert Einstein"</span>,</span><br><span class="line">      <span class="attr">"tags"</span>: [</span><br><span class="line">        <span class="string">"change"</span>,</span><br><span class="line">        <span class="string">"deep-thoughts"</span>,</span><br><span class="line">        <span class="string">"thinking"</span>,</span><br><span class="line">        <span class="string">"world"</span></span><br><span class="line">      ]</span><br><span class="line">    &#125;,</span><br><span class="line">    ...</span><br><span class="line">    &#123;</span><br><span class="line">      <span class="attr">"text"</span>: <span class="string">"“... a mind needs books as a sword needs a whetsto..."</span>,</span><br><span class="line">      <span class="attr">"author"</span>: <span class="string">"George R.R. Martin"</span>,</span><br><span class="line">      <span class="attr">"tags"</span>: [</span><br><span class="line">        <span class="string">"books"</span>,</span><br><span class="line">        <span class="string">"mind"</span></span><br><span class="line">      ]</span><br><span class="line">    &#125;</span><br><span class="line">  ],</span><br><span class="line">  <span class="attr">"items_dropped"</span>: [],</span><br><span class="line">  <span class="attr">"stats"</span>: &#123;</span><br><span class="line">    <span class="attr">"downloader/request_bytes"</span>: <span class="number">2892</span>,</span><br><span class="line">    <span class="attr">"downloader/request_count"</span>: <span class="number">11</span>,</span><br><span class="line">    <span class="attr">"downloader/request_method_count/GET"</span>: <span class="number">11</span>,</span><br><span class="line">    <span class="attr">"downloader/response_bytes"</span>: <span class="number">24812</span>,</span><br><span class="line">    <span class="attr">"downloader/response_count"</span>: <span class="number">11</span>,</span><br><span class="line">    <span class="attr">"downloader/response_status_count/200"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"downloader/response_status_count/404"</span>: <span class="number">1</span>,</span><br><span class="line">    <span class="attr">"dupefilter/filtered"</span>: <span class="number">1</span>,</span><br><span class="line">    <span class="attr">"finish_reason"</span>: <span class="string">"finished"</span>,</span><br><span class="line">    <span class="attr">"finish_time"</span>: <span class="string">"2017-07-12 15:09:02"</span>,</span><br><span class="line">    <span class="attr">"item_scraped_count"</span>: <span class="number">100</span>,</span><br><span class="line">    <span class="attr">"log_count/DEBUG"</span>: <span class="number">112</span>,</span><br><span class="line">    <span class="attr">"log_count/INFO"</span>: <span class="number">8</span>,</span><br><span class="line">    <span class="attr">"memusage/max"</span>: <span class="number">52510720</span>,</span><br><span class="line">    <span class="attr">"memusage/startup"</span>: <span class="number">52510720</span>,</span><br><span class="line">    <span class="attr">"request_depth_max"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"response_received_count"</span>: <span class="number">11</span>,</span><br><span class="line">    <span class="attr">"scheduler/dequeued"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"scheduler/dequeued/memory"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"scheduler/enqueued"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"scheduler/enqueued/memory"</span>: <span class="number">10</span>,</span><br><span class="line">    <span class="attr">"start_time"</span>: <span class="string">"2017-07-12 15:08:56"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"spider_name"</span>: <span class="string">"quotes"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里省略了 items 绝大部分。status 显示了爬取的状态，items 部分是 Scrapy 项目的爬取结果，items_dropped 是被忽略的 Item 列表，stats 是爬取结果的统计情况。此结果和直接运行 Scrapy 项目得到的统计是相同的。 这样一来，我们就通过 HTTP 接口调度 Scrapy 项目并获取爬取结果，如果 Scrapy 项目部署在服务器上，我们可以通过开启一个 Scrapyrt 服务实现任务的调度并直接取到爬取结果，这很方便。</p>
                  <h3 id="5-POST-请求"><a href="#5-POST-请求" class="headerlink" title="5. POST 请求"></a>5. POST 请求</h3>
                  <p>除了 GET 请求，我们还可以通过 POST 请求来请求 Scrapyrt。但是此处 Request Body 必须是一个合法的 JSON 配置，在 JSON 里面可以配置相应的参数，支持的配置参数更多。 目前，JSON 配置支持如下参数。</p>
                  <ul>
                    <li><strong>spider_name</strong>：Spider 名称，字符串类型，必传参数。如果传递的 Spider 名称不存在，则返回 404 错误。</li>
                    <li><strong>max_requests</strong>：最大请求数量，数值类型，可选参数。它定义了 Scrapy 执行请求的 Request 的最大限制，如定义为 5，则表示最多只执行 5 次 Request 请求，其余的则会被忽略。</li>
                    <li><strong>request</strong>：Request 配置，JSON 对象，必传参数。通过该参数可以定义 Request 的各个参数，必须指定 url 字段来指定爬取链接，其他字段可选。</li>
                  </ul>
                  <p>我们看一个 JSON 配置实例，如下所示：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"request"</span>: &#123;</span><br><span class="line">        <span class="attr">"url"</span>: <span class="string">"http://quotes.toscrape.com/"</span>,</span><br><span class="line">        <span class="attr">"callback"</span>: <span class="string">"parse"</span>,</span><br><span class="line">        <span class="attr">"dont_filter"</span>: <span class="string">"True"</span>,</span><br><span class="line">        <span class="attr">"cookies"</span>: &#123;<span class="attr">"foo"</span>: <span class="string">"bar"</span>&#125;</span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attr">"max_requests"</span>: <span class="number">2</span>,</span><br><span class="line">    <span class="attr">"spider_name"</span>: <span class="string">"quotes"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们执行如下命令传递该 Json 配置并发起 POST 请求：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http:<span class="regexp">//</span>localhost:<span class="number">9080</span><span class="regexp">/crawl.json -d '&#123;"request": &#123;"url": "http:/</span><span class="regexp">/quotes.toscrape.com/</span><span class="string">", "</span>dont_filte<span class="string">r": "</span>True<span class="string">", "</span>callback<span class="string">": "</span>parse<span class="string">", "</span>cookies<span class="string">": &#123;"</span>foo<span class="string">": "</span>ba<span class="string">r"&#125;&#125;, "</span>max_requests<span class="string">": 2, "</span>spider_name<span class="string">": "</span>quotes<span class="string">"&#125;'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果和上文类似，同样是输出了爬取状态、结果、统计信息等内容。</p>
                  <h3 id="6-结语"><a href="#6-结语" class="headerlink" title="6. 结语"></a>6. 结语</h3>
                  <p>以上内容便是 Scrapyrt 的相关用法介绍。通过它，我们方便地调度 Scrapy 项目的运行并获取爬取结果。更多的使用方法可以参考官方文档：<a href="http://scrapyrt.readthedocs.io" target="_blank" rel="noopener">http://scrapyrt.readthedocs.io</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-07 10:20:01" itemprop="dateCreated datePublished" datetime="2019-12-07T10:20:01+08:00">2019-12-07</time>
                </span>
                <span id="/8445.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.11–Scrapyrt 的使用" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8443.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/8443.html" class="post-title-link" itemprop="url">Nginx 反向代理返回结果为空的问题</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>最近在开发过程中遇到了这么一个问题： 现在有一个 Web 项目，前端是使用 Vue.js 开发的，整个前端需要部署到 K8S 上，后端和前端分开，同样也需要部署到 K8S 上，因此二者需要打包为 Docker 镜像。 对前端来说，打包 Docker 就遇到了一个问题：跨域访问问题。 因此一个普遍的解决方案就是使用 Nginx 做反向代理。 一般来说，我们需要在打包时配置一下 nginx.conf 文件，然后在 Dockerfile 里面指定即可。</p>
                  <h2 id="Dockerfile"><a href="#Dockerfile" class="headerlink" title="Dockerfile"></a>Dockerfile</h2>
                  <p>首先看下 Dockerfile：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># build stage</span></span><br><span class="line"><span class="keyword">FROM</span> node:lts-alpine as build-stage</span><br><span class="line"><span class="keyword">WORKDIR</span><span class="bash"> /app</span></span><br><span class="line"><span class="keyword">COPY</span><span class="bash"> package*.json ./</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> npm install</span></span><br><span class="line"><span class="keyword">COPY</span><span class="bash"> . .</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> npm run build</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># production stage</span></span><br><span class="line"><span class="keyword">FROM</span> nginx:lts-alpine as production-stage</span><br><span class="line"><span class="keyword">COPY</span><span class="bash"> --from=build-stage /app/dist /usr/share/nginx/html</span></span><br><span class="line"><span class="keyword">COPY</span><span class="bash"> nginx.conf /etc/nginx/conf.d/</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> rm /etc/nginx/conf.d/default.conf </span></span><br><span class="line">    &amp;&amp; mv /etc/nginx/conf.d/nginx.conf /etc/nginx/conf.d/default.conf</span><br><span class="line"><span class="keyword">EXPOSE</span> <span class="number">80</span></span><br><span class="line"><span class="keyword">CMD</span><span class="bash"> [<span class="string">"nginx"</span>, <span class="string">"-g"</span>, <span class="string">"daemon off;"</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>一般来说，对于常规的 Vue.js 前端项目，Dockerfile 就这么写就行了。 简单介绍一下：</p>
                  <ul>
                    <li>第一步，使用 Node.js 镜像，在 Node.js 环境下对项目进行编译，默认会输出到 dist 文件夹下。</li>
                    <li>第二步，使用新的 Nginx 镜像，将编译得到的前端文件拷贝到 nginx 默认 serve 的目录，然后把自定义的 nginx.conf 文件替换为 Nginx 默认的 conf 文件，运行即可。</li>
                  </ul>
                  <h2 id="反向代理"><a href="#反向代理" class="headerlink" title="反向代理"></a>反向代理</h2>
                  <p>这里比较关键的就是 nginx.conf 文件了，为了解决跨域问题，我们一般会将后端的接口进行反向代理。 一般来说，后端的 API 接口都是以 api 为开头的，所以我们需要代理 api 开头的接口地址，nginx.conf 内容一般可以这么写：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">listen</span>       <span class="number">80</span>;</span><br><span class="line">    <span class="attribute">server_name</span>  localhost;</span><br><span class="line"></span><br><span class="line">    <span class="attribute">location</span> /api/ &#123;</span><br><span class="line">                <span class="attribute">proxy_pass</span> http://domain.com/api/;</span><br><span class="line">                <span class="attribute">proxy_set_header</span> X-Forwarded-Proto <span class="variable">$scheme</span>;</span><br><span class="line">                <span class="attribute">proxy_set_header</span> Host <span class="variable">$http_host</span>;</span><br><span class="line">                <span class="attribute">proxy_set_header</span> X-Real-IP <span class="variable">$remote_addr</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="attribute">location</span> / &#123;</span><br><span class="line">        <span class="attribute">root</span>   /usr/share/nginx/html;</span><br><span class="line">        <span class="attribute">index</span>  index.html index.htm;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="attribute">location</span> = /50x.html &#123;</span><br><span class="line">        <span class="attribute">root</span>   /usr/share/nginx/html;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="attribute">error_page</span>  <span class="number">404</span>              /<span class="number">404</span>.html;</span><br><span class="line">    <span class="attribute">error_page</span>   <span class="number">500</span> <span class="number">502</span> <span class="number">503</span> <span class="number">504</span>  /50x.html;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>一般来说，以上的写法是没有问题的，proxy_set_header 也把一些 Header 进行设置，转发到后端服务器。 如果你这么写，打包 Docker 之后，测试没有遇到问题，那就完事了。</p>
                  <h2 id="问题"><a href="#问题" class="headerlink" title="问题"></a>问题</h2>
                  <p>但我遇到了一个奇怪的问题，某个接口在请求的时候，状态码还是 200，但其返回值总是为空，即 Response Data 的内容完全为空。 但是服务器端看 Log 确实有正常返回 Response，使用 Vue 的 devServer 也是正常的，使用 Postman 来请求也是正常的，但是经过 Nginx 这么一反向代理就不行了，什么 Response 都接收不到。 部署到 Prod 环境之后，浏览器上面可以得到这么个错误：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ERR_INCOMPLETE_CHUNKED_ENCODING</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://qiniu.cuiqingcai.com/2019-12-06-202934.png" alt="image-20191207042932549"> 最后经排查，发现后端接口使用时设定了 <code>Transfer-Encoding: chunked</code> 响应头：</p>
                  <figure class="highlight fortran">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">Transfer</span>-Encoding: chunked</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是啥？这时候就需要引出 Keep-Alive 的相关问题了。</p>
                  <h2 id="什么是-Keep-Alive？"><a href="#什么是-Keep-Alive？" class="headerlink" title="什么是 Keep-Alive？"></a>什么是 Keep-Alive？</h2>
                  <p>我们知道 HTTP 协议采用「请求-应答」模式，当使用普通模式，即非 Keep-Alive 模式时，每个请求/应答客户和服务器都要新建一个连接，完成之后立即断开连接（HTTP 协议为无连接的协议）。当使用 Keep-Alive 模式（又称持久连接、连接重用）时，Keep-Alive 功能使客户端到服务器端的连接持续有效，当出现对服务器的后继请求时，Keep-Alive 功能避免了建立或者重新建立连接。</p>
                  <ul>
                    <li>HTTP 1.0 中默认是关闭 Keep-Alive 的，需要在 HTTP 头加入<code>Connection: Keep-Alive</code>，才能启用 Keep-Alive</li>
                    <li>HTTP 1.1 中默认启用 Keep-Alive，如果请求头中加入 <code>Connection: close</code>，Keep-Alive 才关闭。</li>
                  </ul>
                  <p>目前大部分浏览器都是用 HTTP 1.1 协议，也就是说默认都会发起 Keep-Alive 的连接请求了，所以是否能完成一个完整的 Keep-Alive 连接就看服务器设置情况。 启用 Keep-Alive 模式肯定更高效，性能更高。因为避免了建立/释放连接的开销。</p>
                  <h2 id="Keep-Alive-模式下如何传输数据"><a href="#Keep-Alive-模式下如何传输数据" class="headerlink" title="Keep-Alive 模式下如何传输数据"></a>Keep-Alive 模式下如何传输数据</h2>
                  <p>Keep-Alive 模式，客户端如何判断请求所得到的响应数据已经接收完成呢？或者说如何知道服务器已经发生完了数据？ 我们已经知道了，Keep-Alive 模式发送完数据，HTTP 服务器不会自动断开连接，所有不能再使用返回 EOF（-1）来判断。 那么怎么判断呢？一个是使用 Content-Length ，一个是使用 Transfer-Encoding。</p>
                  <h3 id="Content-Length"><a href="#Content-Length" class="headerlink" title="Content-Length"></a>Content-Length</h3>
                  <p>顾名思义，Conent-Length 表示实体内容长度，客户端（服务器）可以根据这个值来判断数据是否接收完成。 由于 <code>Content-Length</code> 字段必须真实反映实体长度，但实际应用中，有些时候实体长度并没那么好获得，例如实体来自于网络文件，或者由动态语言生成。这时候要想准确获取长度，只能开一个足够大的 buffer，等内容全部生成好再计算。但这样做一方面需要更大的内存开销，另一方面也会让客户端等更久。 我们在做 WEB 性能优化时，有一个重要的指标叫 TTFB（Time To First Byte），它代表的是从客户端发出请求到收到响应的第一个字节所花费的时间。大部分浏览器自带的 Network 面板都可以看到这个指标，越短的 TTFB 意味着用户可以越早看到页面内容，体验越好。可想而知，服务端为了计算响应实体长度而缓存所有内容，跟更短的 TTFB 理念背道而驰。但在 HTTP 报文中，实体一定要在头部之后，顺序不能颠倒，为此我们需要一个新的机制：不依赖头部的长度信息，也能知道实体的边界。 但是如果消息中没有 Conent-Length，那该如何来判断呢？又在什么情况下会没有 Conent-Length 呢？</p>
                  <h3 id="Transfer-Encoding"><a href="#Transfer-Encoding" class="headerlink" title="Transfer-Encoding"></a>Transfer-Encoding</h3>
                  <p>当客户端向服务器请求一个静态页面或者一张图片时，服务器可以很清楚地知道内容大小，然后通过 Content-length 消息首部字段告诉客户端需要接收多少数据。但是如果是动态页面等时，服务器是不可能预先知道内容大小，这时就可以使用 分块编码模式来传输数据了。即如果要一边产生数据，一边发给客户端，服务器就需要在请求头中使用<code>Transfer-Encoding: chunked</code> 这样的方式来代替 Content-Length，这就是分块编码。 分块编码相当简单，在头部加入 <code>Transfer-Encoding: chunked</code> 之后，就代表这个报文采用了分块编码。这时，报文中的实体需要改为用一系列分块来传输。每个分块包含十六进制的长度值和数据，长度值独占一行，长度不包括它结尾的 CRLF（rn），也不包括分块数据结尾的 CRLF。最后一个分块长度值必须为 0，对应的分块数据没有内容，表示实体结束。</p>
                  <h2 id="回归问题"><a href="#回归问题" class="headerlink" title="回归问题"></a>回归问题</h2>
                  <p>那么我说了这么一大通有什么用呢？ OK，在我遇到的业务场景中，我发现服务器的响应头中就包含了<code>Transfer-Encoding: chunked</code> 这个字段。 而这个字段，在 HTTP 1.0 是不被支持的。 而 Nginx 的反向代理，默认用的就是 HTTP 1.0，那就导致了数据无法获取的问题，可以参考 Nginx 的官方文档说明：<a href="http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass" target="_blank" rel="noopener">http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass</a>。 原文中：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Syntax: proxy_http_version <span class="number">1.0</span> | <span class="number">1.1</span>;</span><br><span class="line"><span class="keyword">Default</span>: proxy_http_version <span class="number">1.0</span>;</span><br><span class="line"><span class="keyword">By</span> <span class="keyword">default</span>, <span class="keyword">version</span> <span class="number">1.0</span> <span class="keyword">is</span> used. <span class="keyword">Version</span> <span class="number">1.1</span> <span class="keyword">is</span> recommended <span class="keyword">for</span> use <span class="keyword">with</span> keepalive connections <span class="keyword">and</span> NTLM authentication.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以，我们如果要解决这个问题，只需要设置一下 HTTP 版本为 1.1 就好了： 修改 nginx.conf 文件如下：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">location</span> /api/ &#123;</span><br><span class="line">    <span class="attribute">proxy_pass</span> http://domain.com/api/;</span><br><span class="line">    <span class="attribute">proxy_http_version</span> <span class="number">1</span>.<span class="number">1</span>;</span><br><span class="line">    <span class="attribute">proxy_set_header</span> X-Forwarded-Proto <span class="variable">$scheme</span>;</span><br><span class="line">    <span class="attribute">proxy_set_header</span> Host <span class="variable">$http_host</span>;</span><br><span class="line">    <span class="attribute">proxy_set_header</span> X-Real-IP <span class="variable">$remote_addr</span>;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就增加了一行：</p>
                  <figure class="highlight abnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">proxy_http_version <span class="number">1.1</span><span class="comment">;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样再测试，反向代理就会支持 <code>Transfer-Encoding: chunked</code> 模式了，这也就呼应了之前在浏览器中遇到的 ERR_INCOMPLETE_CHUNKED_ENCODING 错误。 自此，问题完美解决。</p>
                  <h2 id="复盘记录"><a href="#复盘记录" class="headerlink" title="复盘记录"></a>复盘记录</h2>
                  <p>一开始本来只想简单一记录就了事的，但一边写，发现某个地方还可以展开写得更详细。 所以干脆最后我对这个问题进行了详细的复盘和记录。在写本文之前，我其实只思考到了 Keep-Alive 和 HTTP 1.1 的问题，其实我对 Transfer-Encoding 这个并没有去深入思考。在边写边总结的过程中，为了把整个脉络讲明白，我又查询了一些 Transfer-Encoding 和 Nginx 的官方文档，对这块的了解变得更加深入，相当于我在整个记录的过程中，又对整个流程梳理了一遍，同时又有额外的收获。 所以，遇到问题，深入去思考、总结和复盘，是很有帮助的，这会让我们对问题的看法和理解更加透彻。 怎么说呢？在开发过程中，难免会遇到一些奇奇怪怪的 Bug，但这其实只是技术问题，总会解决的。 但怎样在开发过程中，不断提高自己的技术能力，我觉得需要从每一个细节出发，去思考一些事情的来龙去脉。思考得越多，我们对整个事件的把握也会越清晰，以后如果再遇到类似的或者关联的事情，就会迎刃而解了。 平时我们可能很多情况下都在写业务代码，可能比较枯燥，感觉对技术没有实质性的提升，但如果我们能从中提炼出一些核心的问题或解决方案，这才是能真正提高技术的时候，这才是最有价值的。</p>
                  <h2 id="参考文章"><a href="#参考文章" class="headerlink" title="参考文章"></a>参考文章</h2>
                  <p>本文部分内容改写或摘自下列内容。</p>
                  <ul>
                    <li>HTTP Keep-Alive模式：<a href="https://www.cnblogs.com/skynet/archive/2010/12/11/1903347.html" target="_blank" rel="noopener">https://www.cnblogs.com/skynet/archive/2010/12/11/1903347.html</a></li>
                    <li>Nginx proxy_set_header 理解：<a href="https://www.jianshu.com/p/cc5167032525" target="_blank" rel="noopener">https://www.jianshu.com/p/cc5167032525</a></li>
                    <li>使用 Docker 打造超溜的前端环境：<a href="https://github.com/axetroy/blog/issues/178" target="_blank" rel="noopener">https://github.com/axetroy/blog/issues/178</a></li>
                    <li>HTTP 协议中的 Transfer-Encoding：<a href="https://imququ.com/post/transfer-encoding-header-in-http.html" target="_blank" rel="noopener">https://imququ.com/post/transfer-encoding-header-in-http.html</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-07 06:26:39" itemprop="dateCreated datePublished" datetime="2019-12-07T06:26:39+08:00">2019-12-07</time>
                </span>
                <span id="/8443.html" class="post-meta-item leancloud_visitors" data-flag-title="Nginx 反向代理返回结果为空的问题" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8418.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 技术杂谈 <i class="label-arrow"></i>
                  </a>
                  <a href="/8418.html" class="post-title-link" itemprop="url">阿里云服务器活动！阿里云代金券 + 1 折优惠码</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>阿里云作为国内最大的云服务商家，个人与企业上云都纷纷首选阿里云。但是在价格方面比整个市场有些许昂贵，让不少用户却而止步。因此<a href="https://www.xingsuyun58.com/" target="_blank" rel="noopener">星速云</a>小编呕心沥血整理阿里云最新优惠折扣【汇总篇】，让大家不用花时间到处寻找优惠信息，帮助站长、开发者和企业们上云购节省项目开支。</p>
                  <hr>
                  <h1 id="最全：阿里云最新优惠获取教程【长期有效】"><a href="#最全：阿里云最新优惠获取教程【长期有效】" class="headerlink" title="最全：阿里云最新优惠获取教程【长期有效】"></a>最全：阿里云最新优惠获取教程【长期有效】</h1>
                  <hr>
                  <h2 id="①：阿里云代金券2000元红包"><a href="#①：阿里云代金券2000元红包" class="headerlink" title="①：阿里云代金券2000元红包"></a>①：阿里云代金券2000元红包</h2>
                  <p>阿里云代金券领取很简单，点击下面链接进行领取。 <a href="https://www.xingsuyun58.com/wp-content/uploads/2019/11/1575037236.png" target="_blank" rel="noopener"><img src="https://www.xingsuyun58.com/wp-content/uploads/2019/11/1575037236.png" alt="阿里云代金券领取和使用步骤教程"></a> 阿里云代金券领取地址：<a href="https://promotion.aliyun.com/ntms/yunparter/invite.html?userCode=ahxhg8oc" target="_blank" rel="noopener">点击领取2000元代金券礼包</a> 点击“立即领取”按钮就可以一键领取到所有满减代金券，最高2000元。别忘记通过购物车一键批量购买哟！</p>
                  <h2 id="②：阿里云9折优惠码"><a href="#②：阿里云9折优惠码" class="headerlink" title="②：阿里云9折优惠码"></a>②：阿里云9折优惠码</h2>
                  <p>新用户还可以使用手机扫码领取一个阿里云9折折扣码叠加上述阿里云代金券使用。该9折码只能通过阿里云手机客户端扫描领取，PC端无法领取，（限ECS首购并且优惠高于7折才可以使用，比如优惠已经为5折，则该折扣码无效） <a href="https://www.xingsuyun58.com/wp-content/uploads/2019/11/15750316201.png" target="_blank" rel="noopener"><img src="https://www.xingsuyun58.com/wp-content/uploads/2019/11/15750316201.png" alt="阿里云代金券"></a> 注明：阿里云9折优惠码与阿里云2000元红包可叠加优惠折扣。</p>
                  <hr>
                  <h1 id="阿里云双12期间（2019-12-3-2019-12-31）最新优惠活动"><a href="#阿里云双12期间（2019-12-3-2019-12-31）最新优惠活动" class="headerlink" title="阿里云双12期间（2019.12.3-2019.12.31）最新优惠活动"></a>阿里云双12期间（2019.12.3-2019.12.31）最新优惠活动</h1>
                  <hr>
                  <p>阿里云双12优惠活动终于开启了，新用户1折甩卖，老用户五折，还可以领取2000元红包，优惠力度不亚于双11优惠活动哟！还不赶紧上云呢？错过双11优惠活动，那么双12不容错过了！ <a href="https://www.aliyun.com/minisite/goods?userCode=ahxhg8oc&amp;share_source=copy_link" target="_blank" rel="noopener"><img src="https://www.xingsuyun58.com/wp-content/uploads/2019/12/aliyun12pintuan.jpg" alt="阿里云双12活动"></a></p>
                  <h2 id="什么？您还不知道云服务器用途"><a href="#什么？您还不知道云服务器用途" class="headerlink" title="什么？您还不知道云服务器用途"></a>什么？您还不知道云服务器用途</h2>
                  <p>不管是做web网站、APP程序后端部署、应用程序后端、小程序后端等，还是打算创业的小伙伴，或者传统IDC自建机房的企业，上云已成为趋势。云服务器更便捷省心、节约IT运维的成本。</p>
                  <h2 id="新用户1折优惠售卖："><a href="#新用户1折优惠售卖：" class="headerlink" title="新用户1折优惠售卖："></a>新用户1折优惠售卖：</h2>
                  <p>实例规格</p>
                  <p>配置</p>
                  <p>带宽</p>
                  <p>时长</p>
                  <p>价格</p>
                  <p>官网购买</p>
                  <p>ECS突发性能型t5</p>
                  <p>1核2G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>89.00元</p>
                  <p>【<a href="https://www.aliyun.com/minisite/goods?userCode=ahxhg8oc&amp;share_source=copy_link" target="_blank" rel="noopener">立即抢购</a>】</p>
                  <p>ECS突发性能型t5</p>
                  <p>1核2G40G高效云盘</p>
                  <p>1M</p>
                  <p>3年</p>
                  <p>229.00元</p>
                  <p>ECS共享型n4</p>
                  <p>2核4G40G高效云盘</p>
                  <p>3M</p>
                  <p>2年</p>
                  <p>469.00元</p>
                  <p>ECS突发性能t5</p>
                  <p>2核4G40G高效云盘</p>
                  <p>5M</p>
                  <p>3年</p>
                  <p>899.00元</p>
                  <p>ECS突发性能t5</p>
                  <p>2核4G40G高效云盘</p>
                  <p>3M</p>
                  <p>3年</p>
                  <p>639.00元</p>
                  <p>ECS共享型n4</p>
                  <p>2核4G40G高效云盘</p>
                  <p>3M</p>
                  <p>3年</p>
                  <p>799.00元</p>
                  <p>ECS共享通用型mn4</p>
                  <p>2核8G40G高效云盘</p>
                  <p>5M</p>
                  <p>3年</p>
                  <p>1399.00元</p>
                  <p>ECS突发性能t5（香港）</p>
                  <p>1核1G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>119.00元</p>
                  <p>ECS网络增强型sn1ne</p>
                  <p>4核8G40G高效云盘</p>
                  <p>5M</p>
                  <p>3年</p>
                  <p>5621.00元</p>
                  <p>8核16G40G高效云盘</p>
                  <p>8M</p>
                  <p>3年</p>
                  <p>12209.00元</p>
                  <hr>
                  <p>注明：突发性t5实例，别看到价格比较便宜就直接购买，里面很多套路，购买页面有提示：限制20%性能基线。释义：依靠CPU 积分来提升 CPU 性能，满足业务需求。当实例实际工作性能高于基准 CPU 计算性能时，会把服务器 CPU 的性能限制在 20%以下，如果这时20%CPU性能满足不了业务需求，云服务器CPU会跑满100%，到那时候你以为是被某大佬攻击了，很有可能是你突发性t5实例CPU 积分消耗完了。笔者建议：如果用户业务对 CPU 要求高的，可以直接略过，选择t5实例（无限制CPU性能）、n4共享型、通用型mn4。以下笔者建议爆款：</p>
                  <h2 id="个人博客与企业微服务首选"><a href="#个人博客与企业微服务首选" class="headerlink" title="个人博客与企业微服务首选"></a>个人博客与企业微服务首选</h2>
                  <p><a href="https://www.aliyun.com/minisite/goods?userCode=ahxhg8oc&amp;share_source=copy_link" target="_blank" rel="noopener"><img src="https://www.xingsuyun58.com/wp-content/uploads/2019/12/15754425651.png" alt="阿里云双12云服务器爆款"></a></p>
                  <h2 id="老用户五折优惠甩卖："><a href="#老用户五折优惠甩卖：" class="headerlink" title="老用户五折优惠甩卖："></a>老用户五折优惠甩卖：</h2>
                  <p>实例规格</p>
                  <p>CPU/内存/云盘</p>
                  <p>带宽</p>
                  <p>时长</p>
                  <p>价格</p>
                  <p>老用户优惠购买</p>
                  <p>云服务器计算型ic5</p>
                  <p>8核8G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>4433.94元</p>
                  <p>【<a href="https://www.aliyun.com/minisite/goods?userCode=ahxhg8oc&amp;share_source=copy_link" target="_blank" rel="noopener">立即抢购</a>】</p>
                  <p>计算网络增强型sn1ne</p>
                  <p>8核16G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>3751.20元</p>
                  <p>通用网络增强型sn2ne</p>
                  <p>8核32G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>5353.20元</p>
                  <p>内存网络增强型se1ne</p>
                  <p>8核64G40G高效云盘</p>
                  <p>1M</p>
                  <p>1年</p>
                  <p>6793.20元</p>
                  <blockquote>
                    <p>注明：本文为星速云原创版权所有，禁止转载，一经发现将追究版权责任！</p>
                  </blockquote>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-06 18:43:17" itemprop="dateCreated datePublished" datetime="2019-12-06T18:43:17+08:00">2019-12-06</time>
                </span>
                <span id="/8418.html" class="post-meta-item leancloud_visitors" data-flag-title="阿里云服务器活动！阿里云代金券 + 1 折优惠码" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8413.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8413.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.10–Scrapy 通用爬虫</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-10-Scrapy-通用爬虫"><a href="#13-10-Scrapy-通用爬虫" class="headerlink" title="13.10 Scrapy 通用爬虫"></a>13.10 Scrapy 通用爬虫</h1>
                  <p>通过 Scrapy，我们可以轻松地完成一个站点爬虫的编写。但如果抓取的站点量非常大，比如爬取各大媒体的新闻信息，多个 Spider 则可能包含很多重复代码。 如果我们将各个站点的 Spider 的公共部分保留下来，不同的部分提取出来作为单独的配置，如爬取规则、页面解析方式等抽离出来做成一个配置文件，那么我们在新增一个爬虫的时候，只需要实现这些网站的爬取规则和提取规则即可。 本节我们就来探究一下 Scrapy 通用爬虫的实现方法。</p>
                  <h3 id="1-CrawlSpider"><a href="#1-CrawlSpider" class="headerlink" title="1. CrawlSpider"></a>1. CrawlSpider</h3>
                  <p>在实现通用爬虫之前我们需要先了解一下 CrawlSpider，其官方文档链接为：<a href="http://scrapy.readthedocs.io/en/latest/topics/spiders.html#crawlspider" target="_blank" rel="noopener">http://scrapy.readthedocs.io/en/latest/topics/spiders.html#crawlspider</a>。 CrawlSpider 是 Scrapy 提供的一个通用 Spider。在 Spider 里，我们可以指定一些爬取规则来实现页面的提取，这些爬取规则由一个专门的数据结构 Rule 表示。Rule 里包含提取和跟进页面的配置，Spider 会根据 Rule 来确定当前页面中的哪些链接需要继续爬取、哪些页面的爬取结果需要用哪个方法解析等。 CrawlSpider 继承自 Spider 类。除了 Spider 类的所有方法和属性，它还提供了一个非常重要的属性和方法。</p>
                  <ul>
                    <li>rules，它是爬取规则属性，是包含一个或多个 Rule 对象的列表。每个 Rule 对爬取网站的动作都做了定义，CrawlSpider 会读取 rules 的每一个 Rule 并进行解析。</li>
                    <li>parse_start_url()，它是一个可重写的方法。当 start_urls 里对应的 Request 得到 Response 时，该方法被调用，它会分析 Response 并必须返回 Item 对象或者 Request 对象。</li>
                  </ul>
                  <p>这里最重要的内容莫过于 Rule 的定义了，它的定义和参数如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">class scrapy.contrib.spiders.Rule(link_extractor, <span class="attribute">callback</span>=None, <span class="attribute">cb_kwargs</span>=None, <span class="attribute">follow</span>=None, <span class="attribute">process_links</span>=None, <span class="attribute">process_request</span>=None)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面对其参数依次说明：</p>
                  <ul>
                    <li>link_extractor，是一个 Link Extractor 对象。通过它，Spider 可以知道从爬取的页面中提取哪些链接。提取出的链接会自动生成 Request。它又是一个数据结构，一般常用 LxmlLinkExtractor 对象作为参数，其定义和参数如下所示：</li>
                  </ul>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">class scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), <span class="attribute">deny_extensions</span>=None, restrict_xpaths=(), restrict_css=(), tags=(<span class="string">'a'</span>, <span class="string">'area'</span>), attrs=(<span class="string">'href'</span>,), <span class="attribute">canonicalize</span>=<span class="literal">False</span>, <span class="attribute">unique</span>=<span class="literal">True</span>, <span class="attribute">process_value</span>=None, <span class="attribute">strip</span>=<span class="literal">True</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>allow 是一个正则表达式或正则表达式列表，它定义了从当前页面提取出的链接哪些是符合要求的，只有符合要求的链接才会被跟进。deny 则相反。allow_domains 定义了符合要求的域名，只有此域名的链接才会被跟进生成新的 Request，它相当于域名白名单。deny_domains 则相反，相当于域名黑名单。restrict_xpaths 定义了从当前页面中 XPath 匹配的区域提取链接，其值是 XPath 表达式或 XPath 表达式列表。restrict_css 定义了从当前页面中 CSS 选择器匹配的区域提取链接，其值是 CSS 选择器或 CSS 选择器列表。还有一些其他参数代表了提取链接的标签、是否去重、链接的处理等内容，使用的频率不高。可以参考文档的参数说明：<a href="http://scrapy.readthedocs.io/en/latest/topics/link-extractors.html#module-scrapy.linkextractors.lxmlhtml" target="_blank" rel="noopener">http://scrapy.readthedocs.io/en/latest/topics/link-extractors.html#module-scrapy.linkextractors.lxmlhtml</a>。</p>
                  <ul>
                    <li>callback，即回调函数，和之前定义 Request 的 callback 有相同的意义。每次从 link_extractor 中获取到链接时，该函数将会调用。该回调函数接收一个 response 作为其第一个参数，并返回一个包含 Item 或 Request 对象的列表。注意，避免使用 parse() 作为回调函数。由于 CrawlSpider 使用 parse() 方法来实现其逻辑，如果 parse() 方法覆盖了，CrawlSpider 将会运行失败。</li>
                    <li>cb_kwargs，字典，它包含传递给回调函数的参数。</li>
                    <li>follow，布尔值，即 True 或 False，它指定根据该规则从 response 提取的链接是否需要跟进。如果 callback 参数为 None，follow 默认设置为 True，否则默认为 False。</li>
                    <li>process_links，指定处理函数，从 link_extractor 中获取到链接列表时，该函数将会调用，它主要用于过滤。</li>
                    <li>process_request，同样是指定处理函数，根据该 Rule 提取到每个 Request 时，该函数都会调用，对 Request 进行处理。该函数必须返回 Request 或者 None。</li>
                  </ul>
                  <p>以上内容便是 CrawlSpider 中的核心 Rule 的基本用法。但这些内容可能还不足以完成一个 CrawlSpider 爬虫。下面我们利用 CrawlSpider 实现新闻网站的爬取实例，来更好地理解 Rule 的用法。</p>
                  <h3 id="2-Item-Loader"><a href="#2-Item-Loader" class="headerlink" title="2. Item Loader"></a>2. Item Loader</h3>
                  <p>我们了解了利用 CrawlSpider 的 Rule 来定义页面的爬取逻辑，这是可配置化的一部分内容。但是，Rule 并没有对 Item 的提取方式做规则定义。对于 Item 的提取，我们需要借助另一个模块 Item Loader 来实现。 Item Loader 提供一种便捷的机制来帮助我们方便地提取 Item。它提供的一系列 API 可以分析原始数据对 Item 进行赋值。Item 提供的是保存抓取数据的容器，而 Item Loader 提供的是填充容器的机制。有了它，数据的提取会变得更加规则化。 Item Loader 的 API 如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">class</span> scrapy.loader.<span class="constructor">ItemLoader([<span class="params">item</span>, <span class="params">selector</span>, <span class="params">response</span>,] <span class="operator">**</span><span class="params">kwargs</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>Item Loader 的 API 返回一个新的 Item Loader 来填充给定的 Item。如果没有给出 Item，则使用 default_item_class 中的类自动实例化。另外，它传入 selector 和 response 参数来使用选择器或响应参数实例化。 下面将依次说明 Item Loader 的 API 参数。</p>
                  <ul>
                    <li>item，Item 对象，可以调用 add_xpath()、add_css() 或 add_value() 等方法来填充 Item 对象。</li>
                    <li>selector，Selector 对象，用来提取填充数据的选择器。</li>
                    <li>response，Response 对象，用于使用构造选择器的 Response。</li>
                  </ul>
                  <p>一个比较典型的 Item Loader 实例如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy.loader import ItemLoader</span><br><span class="line">from project.items import Product</span><br><span class="line"></span><br><span class="line">def parse(self, response):</span><br><span class="line">    loader = <span class="constructor">ItemLoader(<span class="params">item</span>=Product()</span>, response=response)</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">name</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">class</span>=<span class="string">"product_name"</span>]')</span></span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">name</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">class</span>=<span class="string">"product_title"</span>]')</span></span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">price</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">p</span>[@<span class="params">id</span>=<span class="string">"price"</span>]')</span></span><br><span class="line">    loader.add<span class="constructor">_css('<span class="params">stock</span>', '<span class="params">p</span>#<span class="params">stock</span>]')</span></span><br><span class="line">    loader.add<span class="constructor">_value('<span class="params">last_updated</span>', '<span class="params">today</span>')</span></span><br><span class="line">    return loader.load<span class="constructor">_item()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先声明一个 Product Item，用该 Item 和 Response 对象实例化 ItemLoader，调用 add_xpath() 方法把来自两个不同位置的数据提取出来，分配给 name 属性，再用 add_xpath()、add_css()、add_value() 等方法对不同属性依次赋值，最后调用 load_item() 方法实现 Item 的解析。这种方式比较规则化，我们可以把一些参数和规则单独提取出来做成配置文件或存到数据库，即可实现可配置化。 另外，Item Loader 每个字段中都包含了一个 Input Processor（输入处理器）和一个 Output Processor（输出处理器）。Input Processor 收到数据时立刻提取数据，Input Processor 的结果被收集起来并且保存在 ItemLoader 内，但是不分配给 Item。收集到所有的数据后，load_item() 方法被调用来填充再生成 Item 对象。在调用时会先调用 Output Processor 来处理之前收集到的数据，然后再存入 Item 中，这样就生成了 Item。 下面将介绍一些内置的 Processor。</p>
                  <h4 id="Identity"><a href="#Identity" class="headerlink" title="Identity"></a>Identity</h4>
                  <p>Identity 是最简单的 Processor，不进行任何处理，直接返回原来的数据。</p>
                  <h4 id="TakeFirst"><a href="#TakeFirst" class="headerlink" title="TakeFirst"></a>TakeFirst</h4>
                  <p>TakeFirst 返回列表的第一个非空值，类似 extract_first() 的功能，常用作 Output Processor，如下所示：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy<span class="selector-class">.loader</span><span class="selector-class">.processors</span> import TakeFirst</span><br><span class="line">processor = TakeFirst()</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(processor([<span class="string">''</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出结果如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>经过此 Processor 处理后的结果返回了第一个不为空的值。</p>
                  <h4 id="Join"><a href="#Join" class="headerlink" title="Join"></a>Join</h4>
                  <p>Join 方法相当于字符串的 join() 方法，可以把列表拼合成字符串，字符串默认使用空格分隔，如下所示：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy.loader.processors <span class="keyword">import</span> <span class="keyword">Join</span></span><br><span class="line">processor = <span class="keyword">Join</span>()</span><br><span class="line"><span class="keyword">print</span>(processor([<span class="string">'one'</span>, <span class="string">'two'</span>, <span class="string">'three'</span>]))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出结果如下所示：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">one</span> <span class="literal">two</span> <span class="literal">three</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>它也可以通过参数更改默认的分隔符，例如改成逗号：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy.loader.processors <span class="keyword">import</span> <span class="keyword">Join</span></span><br><span class="line">processor = <span class="keyword">Join</span>(<span class="string">','</span>)</span><br><span class="line"><span class="keyword">print</span>(processor([<span class="string">'one'</span>, <span class="string">'two'</span>, <span class="string">'three'</span>]))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下所示：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">one</span>,<span class="literal">two</span>,<span class="literal">three</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="Compose"><a href="#Compose" class="headerlink" title="Compose"></a>Compose</h4>
                  <p>Compose 是用给定的多个函数的组合而构造的 Processor，每个输入值被传递到第一个函数，其输出再传递到第二个函数，依次类推，直到最后一个函数返回整个处理器的输出，如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy.loader.processors import Compose</span><br><span class="line">processor = <span class="constructor">Compose(<span class="params">str</span>.<span class="params">upper</span>, <span class="params">lambda</span> <span class="params">s</span>: <span class="params">s</span>.<span class="params">strip</span>()</span>)</span><br><span class="line">print(processor(' hello world'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">HELLO WORLD</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们构造了一个 Compose Processor，传入一个开头带有空格的字符串。Compose Processor 的参数有两个：第一个是 str.upper，它可以将字母全部转为大写；第二个是一个匿名函数，它调用 strip() 方法去除头尾空白字符。Compose 会顺次调用两个参数，最后返回结果的字符串全部转化为大写并且去除了开头的空格。</p>
                  <h4 id="MapCompose"><a href="#MapCompose" class="headerlink" title="MapCompose"></a>MapCompose</h4>
                  <p>与 Compose 类似，MapCompose 可以迭代处理一个列表输入值，如下所示：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy<span class="selector-class">.loader</span><span class="selector-class">.processors</span> import MapCompose</span><br><span class="line">processor = MapCompose(str<span class="selector-class">.upper</span>, lambda s: s.strip())</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(processor([<span class="string">'Hello'</span>, <span class="string">'World'</span>, <span class="string">'Python'</span>])</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下所示：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'HELLO</span>', <span class="symbol">'WORLD</span>', <span class="symbol">'PYTHON</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>被处理的内容是一个可迭代对象，MapCompose 会将该对象遍历然后依次处理。</p>
                  <h4 id="SelectJmes"><a href="#SelectJmes" class="headerlink" title="SelectJmes"></a>SelectJmes</h4>
                  <p>SelectJmes 可以查询 JSON，传入 Key，返回查询所得的 Value。不过需要先安装 jmespath 库才可以使用它，命令如下所示：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> jmespath</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好 jmespath 之后，便可以使用这个 Processor 了，如下所示：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">scrapy.loader.processors</span> <span class="variable">import</span> <span class="variable">SelectJmes</span></span><br><span class="line"><span class="variable">proc</span> = <span class="function"><span class="title">SelectJmes</span>(<span class="string">'foo'</span>)</span></span><br><span class="line"><span class="variable">processor</span> = <span class="function"><span class="title">SelectJmes</span>(<span class="string">'foo'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">processor</span>(&#123;<span class="string">'foo'</span>: <span class="string">'bar'</span>&#125;))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">bar</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上内容便是一些常用的 Processor，在本节的实例中我们会使用 Processor 来进行数据的处理。 接下来，我们用一个实例来了解 Item Loader 的用法。</p>
                  <h3 id="3-本节目标"><a href="#3-本节目标" class="headerlink" title="3. 本节目标"></a>3. 本节目标</h3>
                  <p>我们以中华网科技类新闻为例，来了解 CrawlSpider 和 Item Loader 的用法，再提取其可配置信息实现可配置化。官网链接为：<a href="http://tech.china.com/。我们需要爬取它的科技类新闻内容，链接为：http://tech.china.com/articles/，页面如图" target="_blank" rel="noopener">http://tech.china.com/。我们需要爬取它的科技类新闻内容，链接为：http://tech.china.com/articles/，页面如图</a> 13-19 所示。 我们要抓取新闻列表中的所有分页的新闻详情，包括标题、正文、时间、来源等信息。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034144.png" alt=""> 图 13-19 爬取站点</p>
                  <h3 id="4-新建项目"><a href="#4-新建项目" class="headerlink" title="4. 新建项目"></a>4. 新建项目</h3>
                  <p>首先新建一个 Scrapy 项目，名为 scrapyuniversal，如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject scrapyuniversal</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>创建一个 CrawlSpider，需要先制定一个模板。我们可以先看看有哪些可用模板，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy genspider -l</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下所示：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">Available</span> templates:</span><br><span class="line">  <span class="keyword">basic</span></span><br><span class="line"><span class="keyword"> </span> crawl</span><br><span class="line">  csvfeed</span><br><span class="line">  xmlfeed</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之前创建 Spider 的时候，我们默认使用了第一个模板 basic。这次要创建 CrawlSpider，就需要使用第二个模板 crawl，创建命令如下所示：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">-t</span> <span class="selector-tag">crawl</span> <span class="selector-tag">china</span> <span class="selector-tag">tech</span><span class="selector-class">.china</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后便会生成一个 CrawlSpider，其内容如下所示：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy.linkextractors <span class="keyword">import</span> LinkExtractor</span><br><span class="line"><span class="keyword">from</span> scrapy.spiders <span class="keyword">import</span> CrawlSpider, Rule</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ChinaSpider</span><span class="params">(CrawlSpider)</span>:</span></span><br><span class="line">    name = <span class="string">'china'</span></span><br><span class="line">    allowed_domains = [<span class="string">'tech.china.com'</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://tech.china.com/'</span>]</span><br><span class="line"></span><br><span class="line">    rules = (Rule(LinkExtractor(allow=<span class="string">r'Items/'</span>), callback=<span class="string">'parse_item'</span>, follow=<span class="literal">True</span>),</span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_item</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        i = &#123;&#125;</span><br><span class="line">        <span class="comment">#i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()</span></span><br><span class="line">        <span class="comment">#i['name'] = response.xpath('//div[@id="name"]').extract()</span></span><br><span class="line">        <span class="comment">#i['description'] = response.xpath('//div[@id="description"]').extract()</span></span><br><span class="line">        <span class="keyword">return</span> i</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这次生成的 Spider 内容多了一个 rules 属性的定义。Rule 的第一个参数是 LinkExtractor，就是上文所说的 LxmlLinkExtractor，只是名称不同。同时，默认的回调函数也不再是 parse，而是 parse_item。</p>
                  <h3 id="5-定义-Rule"><a href="#5-定义-Rule" class="headerlink" title="5. 定义 Rule"></a>5. 定义 Rule</h3>
                  <p>要实现新闻的爬取，我们需要做的就是定义好 Rule，然后实现解析函数。下面我们就来一步步实现这个过程。 首先将 start_urls 修改为起始链接，代码如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">start_urls</span> = [<span class="string">'http://tech.china.com/articles/'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之后，Spider 爬取 start_urls 里面的每一个链接。所以这里第一个爬取的页面就是我们刚才所定义的链接。得到 Response 之后，Spider 就会根据每一个 Rule 来提取这个页面内的超链接，去生成进一步的 Request。接下来，我们就需要定义 Rule 来指定提取哪些链接。 当前页面如图 13-20 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034532.png" alt=""> 图 13-20 页面内容 这是新闻的列表页，下一步自然就是将列表中的每条新闻详情的链接提取出来。这里直接指定这些链接所在区域即可。查看源代码，所有链接都在 ID 为 left_side 的节点内，具体来说是它内部的 class 为 con_item 的节点，如图 13-21 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034527.jpg" alt=""> 图 13-21 列表源码 此处我们可以用 LinkExtractor 的 restrict_xpaths 属性来指定，之后 Spider 就会从这个区域提取所有的超链接并生成 Request。但是，每篇文章的导航中可能还有一些其他的超链接标签，我们只想把需要的新闻链接提取出来。真正的新闻链接路径都是以 article 开头的，我们用一个正则表达式将其匹配出来再赋值给 allow 参数即可。另外，这些链接对应的页面其实就是对应的新闻详情页，而我们需要解析的就是新闻的详情信息，所以此处还需要指定一个回调函数 callback。 到现在我们就可以构造出一个 Rule 了，代码如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="constructor">Rule(LinkExtractor(<span class="params">allow</span>='<span class="params">article</span><span class="operator">/</span>.<span class="operator">*</span>.<span class="params">html</span>', <span class="params">restrict_xpaths</span>='<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"left_side"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">class</span>=<span class="string">"con_item"</span>]')</span>, callback='parse_item')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来，我们还要让当前页面实现分页功能，所以还需要提取下一页的链接。分析网页源码之后可以发现下一页链接是在 ID 为 pageStyle 的节点内，如图 13-22 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034519.jpg" alt=""> 图 13-22 分页源码 但是，下一页节点和其他分页链接区分度不高，要取出此链接我们可以直接用 XPath 的文本匹配方式，所以这里我们直接用 LinkExtractor 的 restrict_xpaths 属性来指定提取的链接即可。另外，我们不需要像新闻详情页一样去提取此分页链接对应的页面详情信息，也就是不需要生成 Item，所以不需要加 callback 参数。另外这下一页的页面如果请求成功了就需要继续像上述情况一样分析，所以它还需要加一个 follow 参数为 True，代表继续跟进匹配分析。其实，follow 参数也可以不加，因为当 callback 为空的时候，follow 默认为 True。此处 Rule 定义为如下所示：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Rule(<span class="name">LinkExtractor</span>(<span class="name">restrict_xpaths=</span>'//div[@id=<span class="string">"pageStyle"</span>]//a[contains(., <span class="string">"下一页"</span>)]'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以现在 rules 就变成了：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">rules = (<span class="constructor">Rule(LinkExtractor(<span class="params">allow</span>='<span class="params">article</span><span class="operator">/</span>.<span class="operator">*</span>.<span class="params">html</span>', <span class="params">restrict_xpaths</span>='<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"left_side"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">class</span>=<span class="string">"con_item"</span>]')</span>, callback='parse_item'),</span><br><span class="line">    <span class="constructor">Rule(LinkExtractor(<span class="params">restrict_xpaths</span>='<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"pageStyle"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">a</span>[<span class="params">contains</span>(., <span class="string">"下一页"</span>)</span>]'))</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接着我们运行一下代码，命令如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl china</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在已经实现页面的翻页和详情页的抓取了，我们仅仅通过定义了两个 Rule 即实现了这样的功能，运行效果如图 13-23 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034515.jpg" alt=""> 图 13-23 运行效果</p>
                  <h3 id="6-解析页面"><a href="#6-解析页面" class="headerlink" title="6. 解析页面"></a>6. 解析页面</h3>
                  <p>接下来我们需要做的就是解析页面内容了，将标题、发布时间、正文、来源提取出来即可。首先定义一个 Item，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Field, Item</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="symbol">NewsItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">title</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">url</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">text</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">datetime</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">source</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">website</span> = <span class="symbol">Field</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的字段分别指新闻标题、链接、正文、发布时间、来源、站点名称，其中站点名称直接赋值为中华网。因为既然是通用爬虫，肯定还有很多爬虫也来爬取同样结构的其他站点的新闻内容，所以需要一个字段来区分一下站点名称。 详情页的预览图如图 13-24 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034510.png" alt=""> 图 13-24 详情页面 如果像之前一样提取内容，就直接调用 response 变量的 xpath()、css() 等方法即可。这里 parse_item() 方法的实现如下所示：</p>
                  <figure class="highlight xquery">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_item(self, response):</span><br><span class="line">    <span class="type">item</span> = NewsItem()</span><br><span class="line">    <span class="type">item</span>[<span class="string">'title'</span>] = response.xpath(<span class="string">'//h1[@id="chan_newsTitle"]/text()'</span>).extract_first()</span><br><span class="line">    <span class="type">item</span>[<span class="string">'url'</span>] = response.url</span><br><span class="line">    <span class="type">item</span>[<span class="string">'text'</span>] = <span class="string">''</span>.join(response.xpath(<span class="string">'//div[@id="chan_newsDetail"]//text()'</span>).extract()).<span class="keyword">strip</span>()</span><br><span class="line">    <span class="type">item</span>[<span class="string">'datetime'</span>] = response.xpath(<span class="string">'//div[@id="chan_newsInfo"]/text()'</span>).re_first(<span class="string">'(d+-d+-d+sd+:d+:d+)'</span>)</span><br><span class="line">    <span class="type">item</span>[<span class="string">'source'</span>] = response.xpath(<span class="string">'//div[@id="chan_newsInfo"]/text()'</span>).re_first(<span class="string">' 来源：(.*)'</span>).<span class="keyword">strip</span>()</span><br><span class="line">    <span class="type">item</span>[<span class="string">'website'</span>] = <span class="string">' 中华网 '</span></span><br><span class="line">    yield <span class="type">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就把每条新闻的信息提取形成了一个 NewsItem 对象。 这时实际上我们就已经完成了 Item 的提取。再运行一下 Spider，如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl china</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出内容如图 13-25 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034503.jpg" alt=""> 图 13-25 输出内容 现在我们就可以成功将每条新闻的信息提取出来。 不过我们发现这种提取方式非常不规整。下面我们再用 Item Loader，通过 add_xpath()、add_css()、add_value() 等方式实现配置化提取。我们可以改写 parse_item()，如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse<span class="constructor">_item(<span class="params">self</span>, <span class="params">response</span>)</span>:</span><br><span class="line">    loader = <span class="constructor">ChinaLoader(<span class="params">item</span>=NewsItem()</span>, response=response)</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">title</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">h1</span>[@<span class="params">id</span>=<span class="string">"chan_newsTitle"</span>]<span class="operator">/</span><span class="params">text</span>()</span>')</span><br><span class="line">    loader.add<span class="constructor">_value('<span class="params">url</span>', <span class="params">response</span>.<span class="params">url</span>)</span></span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">text</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsDetail"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">text</span>()</span>')</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">datetime</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsInfo"</span>]<span class="operator">/</span><span class="params">text</span>()</span>', re='(d+-d+-d+sd+:d+:d+)')</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">source</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsInfo"</span>]<span class="operator">/</span><span class="params">text</span>()</span>', re=' 来源：(.*)')</span><br><span class="line">    loader.add<span class="constructor">_value('<span class="params">website</span>', ' 中华网 ')</span></span><br><span class="line">    yield loader.load<span class="constructor">_item()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了一个 ItemLoader 的子类，名为 ChinaLoader，其实现如下所示：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="title">from</span> scrapy.loader <span class="keyword">import</span> ItemLoader</span><br><span class="line"><span class="title">from</span> scrapy.loader.processors <span class="keyword">import</span> TakeFirst, Join, Compose</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">NewsLoader</span>(<span class="type">ItemLoader</span>):</span></span><br><span class="line"><span class="class">    default_output_processor = <span class="type">TakeFirst</span>()</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">ChinaLoader</span>(<span class="type">NewsLoader</span>):</span></span><br><span class="line"><span class="class">    text_out = <span class="type">Compose</span>(<span class="type">Join</span>(), lambda s: s.strip())</span></span><br><span class="line"><span class="class">    source_out = <span class="type">Compose</span>(<span class="type">Join</span>(), lambda s: s.strip())</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>ChinaLoader 继承了 NewsLoader 类，其内定义了一个通用的 Out Processor 为 TakeFirst，这相当于之前所定义的 extract_first() 方法的功能。我们在 ChinaLoader 中定义了 text_out 和 source_out 字段。这里使用了一个 Compose Processor，它有两个参数：第一个参数 Join 也是一个 Processor，它可以把列表拼合成一个字符串；第二个参数是一个匿名函数，可以将字符串的头尾空白字符去掉。经过这一系列处理之后，我们就将列表形式的提取结果转化为去除头尾空白字符的字符串。 代码重新运行，提取效果是完全一样的。 至此，我们已经实现了爬虫的半通用化配置。</p>
                  <h3 id="7-通用配置抽取"><a href="#7-通用配置抽取" class="headerlink" title="7. 通用配置抽取"></a>7. 通用配置抽取</h3>
                  <p>为什么现在只做到了半通用化？如果我们需要扩展其他站点，仍然需要创建一个新的 CrawlSpider，定义这个站点的 Rule，单独实现 parse_item() 方法。还有很多代码是重复的，如 CrawlSpider 的变量、方法名几乎都是一样的。那么我们可不可以把多个类似的几个爬虫的代码共用，把完全不相同的地方抽离出来，做成可配置文件呢？ 当然可以。那我们可以抽离出哪些部分？所有的变量都可以抽取，如 name、allowed_domains、start_urls、rules 等。这些变量在 CrawlSpider 初始化的时候赋值即可。我们就可以新建一个通用的 Spider 来实现这个功能，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy genspider -t crawl universal universal</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个全新的 Spider 名为 universal。接下来，我们将刚才所写的 Spider 内的属性抽离出来配置成一个 JSON，命名为 china.json，放到 configs 文件夹内，和 spiders 文件夹并列，代码如下所示：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"spider"</span>: <span class="string">"universal"</span>,</span><br><span class="line">  <span class="attr">"website"</span>: <span class="string">"中华网科技"</span>,</span><br><span class="line">  <span class="attr">"type"</span>: <span class="string">"新闻"</span>,</span><br><span class="line">  <span class="attr">"index"</span>: <span class="string">"http://tech.china.com/"</span>,</span><br><span class="line">  <span class="attr">"settings"</span>: &#123;<span class="attr">"USER_AGENT"</span>: <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"start_urls"</span>: [<span class="string">"http://tech.china.com/articles/"</span>],</span><br><span class="line">  <span class="attr">"allowed_domains"</span>: [<span class="string">"tech.china.com"</span>],</span><br><span class="line">  <span class="attr">"rules"</span>: <span class="string">"china"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一个字段 spider 即 Spider 的名称，在这里是 universal。后面是站点的描述，比如站点名称、类型、首页等。随后的 settings 是该 Spider 特有的 settings 配置，如果要覆盖全局项目，settings.py 内的配置可以单独为其配置。随后是 Spider 的一些属性，如 start_urls、allowed_domains、rules 等。rules 也可以单独定义成一个 rules.py 文件，做成配置文件，实现 Rule 的分离，如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy.linkextractors import LinkExtractor</span><br><span class="line">from scrapy.spiders import Rule</span><br><span class="line"></span><br><span class="line">rules = &#123;</span><br><span class="line">    'china': (<span class="constructor">Rule(LinkExtractor(<span class="params">allow</span>='<span class="params">article</span><span class="operator">/</span>.<span class="operator">*</span>.<span class="params">html</span>', <span class="params">restrict_xpaths</span>='<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"left_side"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">class</span>=<span class="string">"con_item"</span>]')</span>,</span><br><span class="line">             callback='parse_item'),</span><br><span class="line">        <span class="constructor">Rule(LinkExtractor(<span class="params">restrict_xpaths</span>='<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"pageStyle"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">a</span>[<span class="params">contains</span>(., <span class="string">"下一页"</span>)</span>]'))</span><br><span class="line">    )</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们将基本的配置抽取出来。如果要启动爬虫，只需要从该配置文件中读取然后动态加载到 Spider 中即可。所以我们需要定义一个读取该 JSON 文件的方法，如下所示：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from os.<span class="built_in">path</span> <span class="keyword">import</span> realpath, dirname</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line">def get_config(<span class="keyword">name</span>):</span><br><span class="line">    <span class="built_in">path</span> = dirname(realpath(__file__)) + <span class="string">'/configs/'</span> + <span class="keyword">name</span> + <span class="string">'.json'</span></span><br><span class="line">    <span class="keyword">with</span> open(<span class="built_in">path</span>, <span class="string">'r'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> f:</span><br><span class="line">        return json.loads(f.read())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>定义了 get_config() 方法之后，我们只需要向其传入 JSON 配置文件的名称即可获取此 JSON 配置信息。随后我们定义入口文件 run.py，把它放在项目根目录下，它的作用是启动 Spider，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import sys</span><br><span class="line"><span class="keyword">from</span> scrapy.utils.project import get_project_settings</span><br><span class="line"><span class="keyword">from</span> scrapyuniversal.spiders.universal import UniversalSpider</span><br><span class="line"><span class="keyword">from</span> scrapyuniversal.utils import get_config</span><br><span class="line"><span class="keyword">from</span> scrapy.crawler import CrawlerProcess</span><br><span class="line"></span><br><span class="line">def <span class="builtin-name">run</span>():</span><br><span class="line">    name = sys.argv[1]</span><br><span class="line">    custom_settings = get_config(name)</span><br><span class="line">    # 爬取使用的 Spider 名称</span><br><span class="line">    spider = custom_settings.<span class="builtin-name">get</span>(<span class="string">'spider'</span>, <span class="string">'universal'</span>)</span><br><span class="line">    project_settings = get_project_settings()</span><br><span class="line">   <span class="built_in"> settings </span>= dict(project_settings.copy())</span><br><span class="line">    # 合并配置</span><br><span class="line">    settings.update(custom_settings.<span class="builtin-name">get</span>(<span class="string">'settings'</span>))</span><br><span class="line">    process = CrawlerProcess(settings)</span><br><span class="line">    # 启动爬虫</span><br><span class="line">    process.crawl(spider, **&#123;<span class="string">'name'</span>: name&#125;)</span><br><span class="line">    process.start()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="builtin-name">run</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行入口为 run()。首先获取命令行的参数并赋值为 name，name 就是 JSON 文件的名称，其实就是要爬取的目标网站的名称。我们首先利用 get_config() 方法，传入该名称读取刚才定义的配置文件。获取爬取使用的 spider 的名称、配置文件中的 settings 配置，然后将获取到的 settings 配置和项目全局的 settings 配置做了合并。新建一个 CrawlerProcess，传入爬取使用的配置。调用 crawl() 和 start() 方法即可启动爬取。 在 universal 中，我们新建一个<strong>init</strong>() 方法，进行初始化配置，实现如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy.linkextractors import LinkExtractor</span><br><span class="line"><span class="keyword">from</span> scrapy.spiders import CrawlSpider, Rule</span><br><span class="line"><span class="keyword">from</span> scrapyuniversal.utils import get_config</span><br><span class="line"><span class="keyword">from</span> scrapyuniversal.rules import rules</span><br><span class="line"></span><br><span class="line">class UniversalSpider(CrawlSpider):</span><br><span class="line">    name = <span class="string">'universal'</span></span><br><span class="line">    def __init__(self, name, <span class="number">*a</span>rgs, **kwargs):</span><br><span class="line">       <span class="built_in"> config </span>= get_config(name)</span><br><span class="line">        self.config = config</span><br><span class="line">        self.rules = rules.<span class="builtin-name">get</span>(config.<span class="builtin-name">get</span>(<span class="string">'rules'</span>))</span><br><span class="line">        self.start_urls = config.<span class="builtin-name">get</span>(<span class="string">'start_urls'</span>)</span><br><span class="line">        self.allowed_domains = config.<span class="builtin-name">get</span>(<span class="string">'allowed_domains'</span>)</span><br><span class="line">        super(UniversalSpider, self).__init__(<span class="number">*a</span>rgs, **kwargs)</span><br><span class="line"></span><br><span class="line">    def parse_item(self, response):</span><br><span class="line">        i = &#123;&#125;</span><br><span class="line">        return i</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 <strong>init</strong>() 方法中，start_urls、allowed_domains、rules 等属性被赋值。其中，rules 属性另外读取了 rules.py 的配置，这样就成功实现爬虫的基础配置。 接下来，执行如下命令运行爬虫：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">python3 <span class="keyword">run</span>.<span class="bash">py china</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>程序会首先读取 JSON 配置文件，将配置中的一些属性赋值给 Spider，然后启动爬取。运行效果完全相同，运行结果如图 13-26 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034600.jpg" alt=""> 图 13-26 运行结果 现在我们已经对 Spider 的基础属性实现了可配置化。剩下的解析部分同样需要实现可配置化，原来的解析函数如下所示：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse<span class="constructor">_item(<span class="params">self</span>, <span class="params">response</span>)</span>:</span><br><span class="line">    loader = <span class="constructor">ChinaLoader(<span class="params">item</span>=NewsItem()</span>, response=response)</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">title</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">h1</span>[@<span class="params">id</span>=<span class="string">"chan_newsTitle"</span>]<span class="operator">/</span><span class="params">text</span>()</span>')</span><br><span class="line">    loader.add<span class="constructor">_value('<span class="params">url</span>', <span class="params">response</span>.<span class="params">url</span>)</span></span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">text</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsDetail"</span>]<span class="operator">/</span><span class="operator">/</span><span class="params">text</span>()</span>')</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">datetime</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsInfo"</span>]<span class="operator">/</span><span class="params">text</span>()</span>', re='(d+-d+-d+sd+:d+:d+)')</span><br><span class="line">    loader.add<span class="constructor">_xpath('<span class="params">source</span>', '<span class="operator">/</span><span class="operator">/</span><span class="params">div</span>[@<span class="params">id</span>=<span class="string">"chan_newsInfo"</span>]<span class="operator">/</span><span class="params">text</span>()</span>', re=' 来源：(.*)')</span><br><span class="line">    loader.add<span class="constructor">_value('<span class="params">website</span>', ' 中华网 ')</span></span><br><span class="line">    yield loader.load<span class="constructor">_item()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们需要将这些配置也抽离出来。这里的变量主要有 Item Loader 类的选用、Item 类的选用、Item Loader 方法参数的定义，我们可以在 JSON 文件中添加如下 item 的配置：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">"item"</span>: &#123;</span><br><span class="line">  <span class="string">"class"</span>: <span class="string">"NewsItem"</span>,</span><br><span class="line">  <span class="string">"loader"</span>: <span class="string">"ChinaLoader"</span>,</span><br><span class="line">  <span class="string">"attrs"</span>: &#123;</span><br><span class="line">    <span class="string">"title"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"xpath"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"//h1[@id='chan_newsTitle']/text()"</span>]</span><br><span class="line">      &#125;</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">"url"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"attr"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"url"</span>]</span><br><span class="line">      &#125;</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">"text"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"xpath"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"//div[@id='chan_newsDetail']//text()"</span>]</span><br><span class="line">      &#125;</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">"datetime"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"xpath"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"//div[@id='chan_newsInfo']/text()"</span>],</span><br><span class="line">        <span class="string">"re"</span>: <span class="string">"(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"</span></span><br><span class="line">      &#125;</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">"source"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"xpath"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"//div[@id='chan_newsInfo']/text()"</span>],</span><br><span class="line">        <span class="string">"re"</span>: <span class="string">"来源：(.*)"</span></span><br><span class="line">      &#125;</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">"website"</span>: [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="string">"method"</span>: <span class="string">"value"</span>,</span><br><span class="line">        <span class="string">"args"</span>: [<span class="string">"中华网"</span>]</span><br><span class="line">      &#125;</span><br><span class="line">    ]</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了 class 和 loader 属性，它们分别代表 Item 和 Item Loader 所使用的类。定义了 attrs 属性来定义每个字段的提取规则，例如，title 定义的每一项都包含一个 method 属性，它代表使用的提取方法，如 xpath 即代表调用 Item Loader 的 add_xpath() 方法。args 即参数，就是 add_xpath() 的第二个参数，即 XPath 表达式。针对 datetime 字段，我们还用了一次正则提取，所以这里还可以定义一个 re 参数来传递提取时所使用的正则表达式。 我们还要将这些配置之后动态加载到 parse_item() 方法里。最后，最重要的就是实现 parse_item() 方法，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_item(self, response):</span><br><span class="line">   item = self.config.<span class="builtin-name">get</span>(<span class="string">'item'</span>)</span><br><span class="line">   <span class="keyword">if</span> item:</span><br><span class="line">       cls = eval(item.<span class="builtin-name">get</span>(<span class="string">'class'</span>))()</span><br><span class="line">       loader = eval(item.<span class="builtin-name">get</span>(<span class="string">'loader'</span>))(cls, <span class="attribute">response</span>=response)</span><br><span class="line">       # 动态获取属性配置</span><br><span class="line">       <span class="keyword">for</span> key, value <span class="keyword">in</span> item.<span class="builtin-name">get</span>(<span class="string">'attrs'</span>).items():</span><br><span class="line">           <span class="keyword">for</span> extractor <span class="keyword">in</span> value:</span><br><span class="line">               <span class="keyword">if</span> extractor.<span class="builtin-name">get</span>(<span class="string">'method'</span>) == <span class="string">'xpath'</span>:</span><br><span class="line">                   loader.add_xpath(key, <span class="number">*e</span>xtractor.<span class="builtin-name">get</span>(<span class="string">'args'</span>), **&#123;<span class="string">'re'</span>: extractor.<span class="builtin-name">get</span>(<span class="string">'re'</span>)&#125;)</span><br><span class="line">               <span class="keyword">if</span> extractor.<span class="builtin-name">get</span>(<span class="string">'method'</span>) == <span class="string">'css'</span>:</span><br><span class="line">                   loader.add_css(key, <span class="number">*e</span>xtractor.<span class="builtin-name">get</span>(<span class="string">'args'</span>), **&#123;<span class="string">'re'</span>: extractor.<span class="builtin-name">get</span>(<span class="string">'re'</span>)&#125;)</span><br><span class="line">               <span class="keyword">if</span> extractor.<span class="builtin-name">get</span>(<span class="string">'method'</span>) == <span class="string">'value'</span>:</span><br><span class="line">                   loader.add_value(key, <span class="number">*e</span>xtractor.<span class="builtin-name">get</span>(<span class="string">'args'</span>), **&#123;<span class="string">'re'</span>: extractor.<span class="builtin-name">get</span>(<span class="string">'re'</span>)&#125;)</span><br><span class="line">               <span class="keyword">if</span> extractor.<span class="builtin-name">get</span>(<span class="string">'method'</span>) == <span class="string">'attr'</span>:</span><br><span class="line">                   loader.add_value(key, getattr(response, <span class="number">*e</span>xtractor.<span class="builtin-name">get</span>(<span class="string">'args'</span>)))</span><br><span class="line">       yield loader.load_item()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先获取 Item 的配置信息，然后获取 class 的配置，将其初始化，初始化 Item Loader，遍历 Item 的各个属性依次进行提取。判断 method 字段，调用对应的处理方法进行处理。如 method 为 css，就调用 Item Loader 的 add_css() 方法进行提取。所有配置动态加载完毕之后，调用 load_item() 方法将 Item 提取出来。 重新运行程序，结果如图 13-27 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034606.jpg" alt=""> 图 13-27 运行结果 运行结果是完全相同的。 我们再回过头看一下 start_urls 的配置。这里 start_urls 只可以配置具体的链接。如果这些链接有 100 个、1000 个，我们总不能将所有的链接全部列出来吧？在某些情况下，start_urls 也需要动态配置。我们将 start_urls 分成两种，一种是直接配置 URL 列表，一种是调用方法生成，它们分别定义为 static 和 dynamic 类型。 本例中的 start_urls 很明显是 static 类型的，所以 start_urls 配置改写如下所示： ```json”start_urls”: {“type”:”static”,”value”: [“<a href="http://tech.china.com/articles/" target="_blank" rel="noopener">http://tech.china.com/articles/</a>“] }</p>
                  <figure class="highlight autohotkey">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">如果 start_urls 是动态生成的，我们可以调用方法传参数，如下所示：</span><br><span class="line">```json</span><br><span class="line"><span class="string">"start_urls"</span>: &#123;</span><br><span class="line">  <span class="string">"type"</span>: <span class="string">"dynamic"</span>,</span><br><span class="line">  <span class="string">"method"</span>: <span class="string">"china"</span>,</span><br><span class="line">  <span class="string">"args"</span>: [<span class="number">5</span>, <span class="number">10</span>]</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 start_urls 定义为 dynamic 类型，指定方法为 urls_china()，然后传入参数 5 和 10，来生成第 5 到 10 页的链接。这样我们只需要实现该方法即可，统一新建一个 urls.py 文件，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def china(start, end):</span><br><span class="line">    <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(start, end + 1):</span><br><span class="line">        yield <span class="string">'http://tech.china.com/articles/index_'</span> + str(page) + <span class="string">'.html'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其他站点可以自行配置。如某些链接需要用到时间戳，加密参数等，均可通过自定义方法实现。 接下来在 Spider 的 <strong>init</strong>() 方法中，start_urls 的配置改写如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapyuniversal import urls</span><br><span class="line"></span><br><span class="line">start_urls = config.<span class="builtin-name">get</span>(<span class="string">'start_urls'</span>)</span><br><span class="line"><span class="keyword">if</span> start_urls:</span><br><span class="line">    <span class="keyword">if</span> start_urls.<span class="builtin-name">get</span>(<span class="string">'type'</span>) == <span class="string">'static'</span>:</span><br><span class="line">        self.start_urls = start_urls.<span class="builtin-name">get</span>(<span class="string">'value'</span>)</span><br><span class="line">    elif start_urls.<span class="builtin-name">get</span>(<span class="string">'type'</span>) == <span class="string">'dynamic'</span>:</span><br><span class="line">        self.start_urls = list(eval(<span class="string">'urls.'</span> + start_urls.<span class="builtin-name">get</span>(<span class="string">'method'</span>))(*start_urls.<span class="builtin-name">get</span>(<span class="string">'args'</span>, [])))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里通过判定 start_urls 的类型分别进行不同的处理，这样我们就可以实现 start_urls 的配置了。 至此，Spider 的设置、起始链接、属性、提取方法都已经实现了全部的可配置化。 综上所述，整个项目的配置包括如下内容。</p>
                  <ul>
                    <li>spider，指定所使用的 Spider 的名称。</li>
                    <li>settings，可以专门为 Spider 定制配置信息，会覆盖项目级别的配置。</li>
                    <li>start_urls，指定爬虫爬取的起始链接。</li>
                    <li>allowed_domains，允许爬取的站点。</li>
                    <li>rules，站点的爬取规则。</li>
                    <li>item，数据的提取规则。</li>
                  </ul>
                  <p>我们实现了 Scrapy 的通用爬虫，每个站点只需要修改 JSON 文件即可实现自由配置。</p>
                  <h3 id="7-本节代码"><a href="#7-本节代码" class="headerlink" title="7. 本节代码"></a>7. 本节代码</h3>
                  <p>本节代码地址为：<a href="https://github.com/Python3WebSpider/ScrapyUniversal" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapyUniversal</a>。</p>
                  <h3 id="8-结语"><a href="#8-结语" class="headerlink" title="8. 结语"></a>8. 结语</h3>
                  <p>本节介绍了 Scrapy 通用爬虫的实现。我们将所有配置抽离出来，每增加一个爬虫，就只需要增加一个 JSON 文件配置。之后我们只需要维护这些配置文件即可。如果要更加方便的管理，可以将规则存入数据库，再对接可视化管理页面即可。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-06 09:30:05" itemprop="dateCreated datePublished" datetime="2019-12-06T09:30:05+08:00">2019-12-06</time>
                </span>
                <span id="/8413.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.10–Scrapy 通用爬虫" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>18k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>16 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8410.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8410.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.9–Scrapy 对接 Splash</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-9-Scrapy-对接-Splash"><a href="#13-9-Scrapy-对接-Splash" class="headerlink" title="13.9 Scrapy 对接 Splash"></a>13.9 Scrapy 对接 Splash</h1>
                  <p>在上一节我们实现了 Scrapy 对接 Selenium 抓取淘宝商品的过程，这是一种抓取 JavaScript 动态渲染页面的方式。除了 Selenium，Splash 也可以实现同样的功能。本节我们来了解 Scrapy 对接 Splash 来进行页面抓取的方式。</p>
                  <h3 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h3>
                  <p>请确保 Splash 已经正确安装并正常运行，同时安装好 Scrapy-Splash 库，如果没有安装可以参考第 1 章的安装说明。</p>
                  <h3 id="2-新建项目"><a href="#2-新建项目" class="headerlink" title="2. 新建项目"></a>2. 新建项目</h3>
                  <p>首先新建一个项目，名为 scrapysplashtest，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject scrapysplashtest</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建一个 Spider，命令如下所示：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">taobao</span> <span class="selector-tag">www</span><span class="selector-class">.taobao</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="3-添加配置"><a href="#3-添加配置" class="headerlink" title="3. 添加配置"></a>3. 添加配置</h3>
                  <p>可以参考 Scrapy-Splash 的配置说明进行一步步的配置，链接如下：<a href="https://github.com/scrapy-plugins/scrapy-splash#configuration" target="_blank" rel="noopener">https://github.com/scrapy-plugins/scrapy-splash#configuration</a>。 修改 settings.py，配置 SPLASH_URL。在这里我们的 Splash 是在本地运行的，所以可以直接配置本地的地址：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SPLASH_URL</span> = <span class="string">'http://localhost:8050'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果 Splash 是在远程服务器运行的，那此处就应该配置为远程的地址。例如运行在 IP 为 120.27.34.25 的服务器上，则此处应该配置为：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SPLASH_URL</span> = <span class="string">'http://120.27.34.25:8050'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>还需要配置几个 Middleware，代码如下所示：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">DOWNLOADER_MIDDLEWARES</span> <span class="string">=</span> <span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'scrapy_splash.SplashCookiesMiddleware':</span> <span class="number">723</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy_splash.SplashMiddleware':</span> <span class="number">725</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':</span> <span class="number">810</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br><span class="line"><span class="string">SPIDER_MIDDLEWARES</span> <span class="string">=</span> <span class="string">&#123;'scrapy_splash.SplashDeduplicateArgsMiddleware':</span> <span class="number">100</span><span class="string">,&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里配置了三个 Downloader Middleware 和一个 Spider Middleware，这是 Scrapy-Splash 的核心部分。我们不再需要像对接 Selenium 那样实现一个 Downloader Middleware，Scrapy-Splash 库都为我们准备好了，直接配置即可。 还需要配置一个去重的类 DUPEFILTER_CLASS，代码如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">DUPEFILTER_CLASS</span> = <span class="string">'scrapy_splash.SplashAwareDupeFilter'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后配置一个 Cache 存储 HTTPCACHE_STORAGE，代码如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">HTTPCACHE_STORAGE</span> = <span class="string">'scrapy_splash.SplashAwareFSCacheStorage'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="4-新建请求"><a href="#4-新建请求" class="headerlink" title="4. 新建请求"></a>4. 新建请求</h3>
                  <p>配置完成之后，我们就可以利用 Splash 来抓取页面了。我们可以直接生成一个 SplashRequest 对象并传递相应的参数，Scrapy 会将此请求转发给 Splash，Splash 对页面进行渲染加载，然后再将渲染结果传递回来。此时 Response 的内容就是渲染完成的页面结果了，最后交给 Spider 解析即可。 我们来看一个示例，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yield SplashRequest(url, self.parse_result,</span><br><span class="line">    args=&#123;</span><br><span class="line">        # optional; parameters passed <span class="keyword">to</span> Splash HTTP API</span><br><span class="line">        <span class="string">'wait'</span>: 0.5,</span><br><span class="line">        # <span class="string">'url'</span> is prefilled <span class="keyword">from</span> request url</span><br><span class="line">        # <span class="string">'http_method'</span> is <span class="builtin-name">set</span> <span class="keyword">to</span> <span class="string">'POST'</span> <span class="keyword">for</span> POST requests</span><br><span class="line">        # <span class="string">'body'</span> is <span class="builtin-name">set</span> <span class="keyword">to</span> request body <span class="keyword">for</span> POST requests</span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attribute">endpoint</span>=<span class="string">'render.json'</span>, # optional;<span class="built_in"> default </span>is render.html</span><br><span class="line">    <span class="attribute">splash_url</span>=<span class="string">'&lt;url&gt;'</span>,     # optional; overrides SPLASH_URL</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里构造了一个 SplashRequest 对象，前两个参数依然是请求的 URL 和回调函数，另外还可以通过 args 传递一些渲染参数，例如等待时间 wait 等，还可以根据 endpoint 参数指定渲染接口，另外还有更多的参数可以参考文档的说明：<a href="https://github.com/scrapy-plugins/scrapy-splash#requests" target="_blank" rel="noopener">https://github.com/scrapy-plugins/scrapy-splash#requests</a>。 另外我们也可以生成 Request 对象，关于 Splash 的配置通过 meta 属性配置即可，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yield scrapy.Request(url, self.parse_result, meta=&#123;</span><br><span class="line">    <span class="string">'splash'</span>: &#123;</span><br><span class="line">        <span class="string">'args'</span>: &#123;</span><br><span class="line">            # <span class="builtin-name">set</span> rendering arguments here</span><br><span class="line">            <span class="string">'html'</span>: 1,</span><br><span class="line">            <span class="string">'png'</span>: 1,</span><br><span class="line">            # <span class="string">'url'</span> is prefilled <span class="keyword">from</span> request url</span><br><span class="line">            # <span class="string">'http_method'</span> is <span class="builtin-name">set</span> <span class="keyword">to</span> <span class="string">'POST'</span> <span class="keyword">for</span> POST requests</span><br><span class="line">            # <span class="string">'body'</span> is <span class="builtin-name">set</span> <span class="keyword">to</span> request body <span class="keyword">for</span> POST requests</span><br><span class="line">        &#125;,</span><br><span class="line">        # optional parameters</span><br><span class="line">        <span class="string">'endpoint'</span>: <span class="string">'render.json'</span>,  # optional;<span class="built_in"> default </span>is render.json</span><br><span class="line">        <span class="string">'splash_url'</span>: <span class="string">'&lt;url&gt;'</span>,      # optional; overrides SPLASH_URL</span><br><span class="line">        <span class="string">'slot_policy'</span>: scrapy_splash.SlotPolicy.PER_DOMAIN,</span><br><span class="line">        <span class="string">'splash_headers'</span>: &#123;&#125;,       # optional; a dict with headers sent <span class="keyword">to</span> Splash</span><br><span class="line">        <span class="string">'dont_process_response'</span>: <span class="literal">True</span>, # optional,<span class="built_in"> default </span>is <span class="literal">False</span></span><br><span class="line">        <span class="string">'dont_send_headers'</span>: <span class="literal">True</span>,  # optional,<span class="built_in"> default </span>is <span class="literal">False</span></span><br><span class="line">        <span class="string">'magic_response'</span>: <span class="literal">False</span>,    # optional,<span class="built_in"> default </span>is <span class="literal">True</span></span><br><span class="line">    &#125;</span><br><span class="line">&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>SplashRequest 对象通过 args 来配置和 Request 对象通过 meta 来配置，两种方式达到的效果是相同的。 本节我们要做的抓取是淘宝商品信息，涉及页面加载等待、模拟点击翻页等操作。我们可以首先定义一个 Lua 脚本，来实现页面加载、模拟点击翻页的功能，代码如下所示：</p>
                  <figure class="highlight irpf90">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span></span> main(splash, args)</span><br><span class="line">  args = &#123;</span><br><span class="line">    url=<span class="string">"https://s.taobao.com/search?q=iPad"</span>,</span><br><span class="line">    <span class="keyword">wait</span>=<span class="number">5</span>,</span><br><span class="line">    page=<span class="number">5</span></span><br><span class="line">  &#125;</span><br><span class="line">  splash.images_enabled = false</span><br><span class="line">  <span class="keyword">assert</span>(splash:go(args.url))</span><br><span class="line">  <span class="keyword">assert</span>(splash:<span class="keyword">wait</span>(args.<span class="keyword">wait</span>))</span><br><span class="line">  js = string.<span class="keyword">format</span>(<span class="string">"document.querySelector('#mainsrp-pager div.form&gt; input').value=% d;document.querySelector('#mainsrp-pager div.form&gt; span.btn.J_Submit').click()"</span>, args.page)</span><br><span class="line">  splash:evaljs(js)</span><br><span class="line">  <span class="keyword">assert</span>(splash:<span class="keyword">wait</span>(args.<span class="keyword">wait</span>))</span><br><span class="line">  <span class="keyword">return</span> splash:png()</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们定义了三个参数：请求的链接 url、等待时间 wait、分页页码 page。然后禁用图片加载，请求淘宝的商品列表页面，通过 evaljs() 方法调用 JavaScript 代码，实现页码填充和翻页点击，最后返回页面截图。我们将脚本放到 Splash 中运行，正常获取到页面截图，如图 13-15 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034117.jpg" alt=""> 图 13-15 页面截图 翻页操作也成功实现，如图 13-16 所示即为当前页码，和我们传入的页码 page 参数是相同的。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034121.jpg" alt=""> 图 13-16 翻页结果 我们只需要在 Spider 里用 SplashRequest 对接 Lua 脚本就好了，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy import Spider</span><br><span class="line"><span class="keyword">from</span> urllib.parse import quote</span><br><span class="line"><span class="keyword">from</span> scrapysplashtest.items import ProductItem</span><br><span class="line"><span class="keyword">from</span> scrapy_splash import SplashRequest</span><br><span class="line"></span><br><span class="line">script = <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">function main(splash, args)</span></span><br><span class="line"><span class="string">  splash.images_enabled = false</span></span><br><span class="line"><span class="string">  assert(splash:go(args.url))</span></span><br><span class="line"><span class="string">  assert(splash:wait(args.wait))</span></span><br><span class="line"><span class="string">  js = string.format("</span>document.querySelector(<span class="string">'#mainsrp-pager div.form&gt; input'</span>).<span class="attribute">value</span>=% d;document.querySelector(<span class="string">'#mainsrp-pager div.form&gt; span.btn.J_Submit'</span>).click()<span class="string">", args.page)</span></span><br><span class="line"><span class="string">  splash:evaljs(js)</span></span><br><span class="line"><span class="string">  assert(splash:wait(args.wait))</span></span><br><span class="line"><span class="string">  return splash:html()</span></span><br><span class="line"><span class="string">end</span></span><br><span class="line"><span class="string">"</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line">class TaobaoSpider(Spider):</span><br><span class="line">    name = <span class="string">'taobao'</span></span><br><span class="line">    allowed_domains = [<span class="string">'www.taobao.com'</span>]</span><br><span class="line">    base_url = <span class="string">'https://s.taobao.com/search?q='</span></span><br><span class="line"></span><br><span class="line">    def start_requests(self):</span><br><span class="line">        <span class="keyword">for</span> keyword <span class="keyword">in</span> self.settings.<span class="builtin-name">get</span>(<span class="string">'KEYWORDS'</span>):</span><br><span class="line">            <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, self.settings.<span class="builtin-name">get</span>(<span class="string">'MAX_PAGE'</span>) + 1):</span><br><span class="line">                url = self.base_url + quote(keyword)</span><br><span class="line">                yield SplashRequest(url, <span class="attribute">callback</span>=self.parse, <span class="attribute">endpoint</span>=<span class="string">'execute'</span>, args=&#123;<span class="string">'lua_source'</span>: script, <span class="string">'page'</span>: page, <span class="string">'wait'</span>: 7&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们把 Lua 脚本定义成长字符串，通过 SplashRequest 的 args 来传递参数，接口修改为 execute。另外，args 参数里还有一个 lua_source 字段用于指定 Lua 脚本内容。这样我们就成功构造了一个 SplashRequest，对接 Splash 的工作就完成了。 其他的配置不需要更改，Item、Item Pipeline 等设置与上节对接 Selenium 的方式相同，parse() 回调函数也是完全一致的。</p>
                  <h3 id="5-运行"><a href="#5-运行" class="headerlink" title="5. 运行"></a>5. 运行</h3>
                  <p>接下来，我们通过如下命令运行爬虫：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl taobao</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图 13-17 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034129.jpg" alt=""> 图 13-17 运行结果 由于 Splash 和 Scrapy 都支持异步处理，我们可以看到同时会有多个抓取成功的结果。在 Selenium 的对接过程中，每个页面渲染下载是在 Downloader Middleware 里完成的，所以整个过程是阻塞式的。Scrapy 会等待这个过程完成后再继续处理和调度其他请求，这影响了爬取效率。因此使用 Splash 的爬取效率比 Selenium 高很多。 最后我们再看看 MongoDB 的结果，如图 13-18 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034134.jpg" alt=""> 图 13-18 存储结果 结果同样正常保存到了 MongoDB 中。</p>
                  <h3 id="6-本节代码"><a href="#6-本节代码" class="headerlink" title="6. 本节代码"></a>6. 本节代码</h3>
                  <p>本节代码地址：<a href="https://github.com/Python3WebSpider/ScrapySplashTest" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapySplashTest</a>。</p>
                  <h3 id="7-结语"><a href="#7-结语" class="headerlink" title="7. 结语"></a>7. 结语</h3>
                  <p>在 Scrapy 中，建议使用 Splash 处理 JavaScript 动态渲染的页面。这样不会破坏 Scrapy 中的异步处理过程，会大大提高爬取效率。而且 Splash 的安装和配置比较简单，通过 API 调用的方式实现了模块分离，大规模爬取的部署也更加方便。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-06 09:27:03" itemprop="dateCreated datePublished" datetime="2019-12-06T09:27:03+08:00">2019-12-06</time>
                </span>
                <span id="/8410.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.9–Scrapy 对接 Splash" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8397.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8397.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.8–Scrapy 对接 Selenium</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-8-Scrapy-对接-Selenium"><a href="#13-8-Scrapy-对接-Selenium" class="headerlink" title="13.8 Scrapy 对接 Selenium"></a>13.8 Scrapy 对接 Selenium</h1>
                  <p>Scrapy 抓取页面的方式和 requests 库类似，都是直接模拟 HTTP 请求，而 Scrapy 也不能抓取 JavaScript 动态渲染的页面。在前文中抓取 JavaScript 渲染的页面有两种方式。一种是分析 Ajax 请求，找到其对应的接口抓取，Scrapy 同样可以用此种方式抓取。另一种是直接用 Selenium 或 Splash 模拟浏览器进行抓取，我们不需要关心页面后台发生的请求，也不需要分析渲染过程，只需要关心页面最终结果即可，可见即可爬。那么，如果 Scrapy 可以对接 Selenium，那 Scrapy 就可以处理任何网站的抓取了。</p>
                  <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                  <p>本节我们来看看 Scrapy 框架如何对接 Selenium，以 PhantomJS 进行演示。我们依然抓取淘宝商品信息，抓取逻辑和前文中用 Selenium 抓取淘宝商品完全相同。</p>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>请确保 PhantomJS 和 MongoDB 已经安装好并可以正常运行，安装好 Scrapy、Selenium、PyMongo 库，安装方式可以参考第 1 章的安装说明。</p>
                  <h3 id="3-新建项目"><a href="#3-新建项目" class="headerlink" title="3. 新建项目"></a>3. 新建项目</h3>
                  <p>首先新建项目，名为 scrapyseleniumtest，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject scrapyseleniumtest</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建一个 Spider，命令如下所示：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">taobao</span> <span class="selector-tag">www</span><span class="selector-class">.taobao</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>修改 ROBOTSTXT_OBEY 为 False，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">ROBOTSTXT_OBEY</span> = <span class="literal">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="4-定义-Item"><a href="#4-定义-Item" class="headerlink" title="4. 定义 Item"></a>4. 定义 Item</h3>
                  <p>首先定义 Item 对象，名为 ProductItem，代码如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Item, Field</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="symbol">ProductItem</span>(<span class="symbol">Item</span>):</span><br><span class="line"></span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">products</span>'</span><br><span class="line">    <span class="symbol">image</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">price</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">deal</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">title</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">shop</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">location</span> = <span class="symbol">Field</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了 6 个 Field，也就是 6 个字段，跟之前的案例完全相同。然后定义了一个 collection 属性，即此 Item 保存到 MongoDB 的 Collection 名称。 初步实现 Spider 的 start_requests() 方法，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy import Request, Spider</span><br><span class="line"><span class="keyword">from</span> urllib.parse import quote</span><br><span class="line"><span class="keyword">from</span> scrapyseleniumtest.items import ProductItem</span><br><span class="line"></span><br><span class="line">class TaobaoSpider(Spider):</span><br><span class="line">    name = <span class="string">'taobao'</span></span><br><span class="line">    allowed_domains = [<span class="string">'www.taobao.com'</span>]</span><br><span class="line">    base_url = <span class="string">'https://s.taobao.com/search?q='</span></span><br><span class="line"></span><br><span class="line">    def start_requests(self):</span><br><span class="line">        <span class="keyword">for</span> keyword <span class="keyword">in</span> self.settings.<span class="builtin-name">get</span>(<span class="string">'KEYWORDS'</span>):</span><br><span class="line">            <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, self.settings.<span class="builtin-name">get</span>(<span class="string">'MAX_PAGE'</span>) + 1):</span><br><span class="line">                url = self.base_url + quote(keyword)</span><br><span class="line">                yield Request(<span class="attribute">url</span>=url, <span class="attribute">callback</span>=self.parse, meta=&#123;<span class="string">'page'</span>: page&#125;, <span class="attribute">dont_filter</span>=<span class="literal">True</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先定义了一个 base_url，即商品列表的 URL，其后拼接一个搜索关键字就是该关键字在淘宝的搜索结果商品列表页面。 关键字用 KEYWORDS 标识，定义为一个列表。最大翻页页码用 MAX_PAGE 表示。它们统一定义在 setttings.py 里面，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">KEYWORDS</span> = [<span class="string">'iPad'</span>]</span><br><span class="line"><span class="attr">MAX_PAGE</span> = <span class="number">100</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 start_requests() 方法里，我们首先遍历了关键字，遍历了分页页码，构造并生成 Request。由于每次搜索的 URL 是相同的，所以分页页码用 meta 参数来传递，同时设置 dont_filter 不去重。这样爬虫启动的时候，就会生成每个关键字对应的商品列表的每一页的请求了。</p>
                  <h3 id="5-对接-Selenium"><a href="#5-对接-Selenium" class="headerlink" title="5. 对接 Selenium"></a>5. 对接 Selenium</h3>
                  <p>接下来我们需要处理这些请求的抓取。这次我们对接 Selenium 进行抓取，采用 Downloader Middleware 来实现。在 Middleware 里面的 process_request() 方法里对每个抓取请求进行处理，启动浏览器并进行页面渲染，再将渲染后的结果构造一个 HtmlResponse 对象返回。代码实现如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.common.exceptions import TimeoutException</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.by import By</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui import WebDriverWait</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support import expected_conditions as EC</span><br><span class="line"><span class="keyword">from</span> scrapy.http import HtmlResponse</span><br><span class="line"><span class="keyword">from</span><span class="built_in"> logging </span>import getLogger</span><br><span class="line"></span><br><span class="line">class SeleniumMiddleware():</span><br><span class="line">    def __init__(self, <span class="attribute">timeout</span>=None, service_args=[]):</span><br><span class="line">        self.logger = getLogger(__name__)</span><br><span class="line">        self.timeout = timeout</span><br><span class="line">        self.browser = webdriver.PhantomJS(<span class="attribute">service_args</span>=service_args)</span><br><span class="line">        self.browser.set_window_size(1400, 700)</span><br><span class="line">        self.browser.set_page_load_timeout(self.timeout)</span><br><span class="line">        self.wait = WebDriverWait(self.browser, self.timeout)</span><br><span class="line"></span><br><span class="line">    def __del__(self):</span><br><span class="line">        self.browser.close()</span><br><span class="line"></span><br><span class="line">    def process_request(self, request, spider):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        用 PhantomJS 抓取页面</span></span><br><span class="line"><span class="string">        :param request: Request 对象</span></span><br><span class="line"><span class="string">        :param spider: Spider 对象</span></span><br><span class="line"><span class="string">        :return: HtmlResponse</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        self.logger.<span class="builtin-name">debug</span>(<span class="string">'PhantomJS is Starting'</span>)</span><br><span class="line">       <span class="built_in"> page </span>= request.meta.<span class="builtin-name">get</span>(<span class="string">'page'</span>, 1)</span><br><span class="line">        try:</span><br><span class="line">            self.browser.<span class="builtin-name">get</span>(request.url)</span><br><span class="line">            <span class="keyword">if</span><span class="built_in"> page </span>&gt; 1:</span><br><span class="line">                input = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager div.form&gt; input'</span>)))</span><br><span class="line">                submit = self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager div.form&gt; span.btn.J_Submit'</span>)))</span><br><span class="line">                input.clear()</span><br><span class="line">                input.send_keys(page)</span><br><span class="line">                submit.click()</span><br><span class="line">            self.wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager li.item.active&gt; span'</span>), str(page)))</span><br><span class="line">            self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">'.m-itemlist .items .item'</span>)))</span><br><span class="line">            return HtmlResponse(<span class="attribute">url</span>=request.url, <span class="attribute">body</span>=self.browser.page_source, <span class="attribute">request</span>=request, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>, <span class="attribute">status</span>=200)</span><br><span class="line">        except TimeoutException:</span><br><span class="line">            return HtmlResponse(<span class="attribute">url</span>=request.url, <span class="attribute">status</span>=500, <span class="attribute">request</span>=request)</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def from_crawler(cls, crawler):</span><br><span class="line">        return cls(<span class="attribute">timeout</span>=crawler.settings.get('SELENIUM_TIMEOUT'),</span><br><span class="line">                   <span class="attribute">service_args</span>=crawler.settings.get('PHANTOMJS_SERVICE_ARGS'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先我们在 <strong>init</strong>() 里对一些对象进行初始化，包括 PhantomJS、WebDriverWait 等对象，同时设置页面大小和页面加载超时时间。在 process_request() 方法中，我们通过 Request 的 meta 属性获取当前需要爬取的页码，调用 PhantomJS 对象的 get() 方法访问 Request 的对应的 URL。这就相当于从 Request 对象里获取请求链接，然后再用 PhantomJS 加载，而不再使用 Scrapy 里的 Downloader。 随后的处理等待和翻页的方法在此不再赘述，和前文的原理完全相同。最后，页面加载完成之后，我们调用 PhantomJS 的 page_source 属性即可获取当前页面的源代码，然后用它来直接构造并返回一个 HtmlResponse 对象。构造这个对象的时候需要传入多个参数，如 url、body 等，这些参数实际上就是它的基础属性。可以在官方文档查看 HtmlResponse 对象的结构：<a href="https://doc.scrapy.org/en/latest/topics/request-response.html" target="_blank" rel="noopener">https://doc.scrapy.org/en/latest/topics/request-response.html</a>，这样我们就成功利用 PhantomJS 来代替 Scrapy 完成了页面的加载，最后将 Response 返回即可。 有人可能会纳闷：为什么实现这么一个 Downloader Middleware 就可以了？之前的 Request 对象怎么办？Scrapy 不再处理了吗？Response 返回后又传递给了谁？ 是的，Request 对象到这里就不会再处理了，也不会再像以前一样交给 Downloader 下载。Response 会直接传给 Spider 进行解析。 我们需要回顾一下 Downloader Middleware 的 process_request() 方法的处理逻辑，内容如下所示： 当 process_request() 方法返回 Response 对象的时候，更低优先级的 Downloader Middleware 的 process_request() 和 process_exception() 方法就不会被继续调用了，转而开始执行每个 Downloader Middleware 的 process_response() 方法，调用完毕之后直接将 Response 对象发送给 Spider 来处理。 这里直接返回了一个 HtmlResponse 对象，它是 Response 的子类，返回之后便顺次调用每个 Downloader Middleware 的 process_response() 方法。而在 process_response() 中我们没有对其做特殊处理，它会被发送给 Spider，传给 Request 的回调函数进行解析。 到现在，我们应该能了解 Downloader Middleware 实现 Selenium 对接的原理了。 在 settings.py 里，我们设置调用刚才定义的 SeleniumMiddleware、设置等待超时变量 SELENIUM_TIMEOUT、设置 PhantomJS 配置参数 PHANTOMJS_SERVICE_ARGS，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">DOWNLOADER_MIDDLEWARES</span> = &#123;<span class="string">'scrapyseleniumtest.middlewares.SeleniumMiddleware'</span>: <span class="number">543</span>,&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="6-解析页面"><a href="#6-解析页面" class="headerlink" title="6. 解析页面"></a>6. 解析页面</h3>
                  <p>Response 对象就会回传给 Spider 内的回调函数进行解析。所以下一步我们就实现其回调函数，对网页来进行解析，代码如下所示：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">def</span> parse(self, response):</span><br><span class="line">    products = response.xpath(<span class="string">'//div[<span class="variable">@id</span>="mainsrp-itemlist"]//div[<span class="variable">@class</span>="items"][1]//div[contains(<span class="variable">@class</span>, "item")]'</span>)</span><br><span class="line">    for product in products:</span><br><span class="line">        item = ProductItem()</span><br><span class="line">        item[<span class="string">'price'</span>] = <span class="string">''</span>.join(product.xpath(<span class="string">'.//div[contains(<span class="variable">@class</span>, "price")]//text()'</span>).extract()).strip()</span><br><span class="line">        item[<span class="string">'title'</span>] = <span class="string">''</span>.join(product.xpath(<span class="string">'.//div[contains(<span class="variable">@class</span>, "title")]//text()'</span>).extract()).strip()</span><br><span class="line">        item[<span class="string">'shop'</span>] = <span class="string">''</span>.join(product.xpath(<span class="string">'.//div[contains(<span class="variable">@class</span>, "shop")]//text()'</span>).extract()).strip()</span><br><span class="line">        item[<span class="string">'image'</span>] = <span class="string">''</span>.join(product.xpath(<span class="string">'.//div[<span class="variable">@class</span>="pic"]//img[contains(<span class="variable">@class</span>, "img")]/<span class="variable">@data</span>-src'</span>).extract()).strip()</span><br><span class="line">        item[<span class="string">'deal'</span>] = product.xpath(<span class="string">'.//div[contains(<span class="variable">@class</span>, "deal-cnt")]//text()'</span>).extract_first()</span><br><span class="line">        item[<span class="string">'location'</span>] = product.xpath(<span class="string">'.//div[contains(<span class="variable">@class</span>, "location")]//text()'</span>).extract_first()</span><br><span class="line">        yield item</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们使用 XPath 进行解析，调用 response 变量的 xpath() 方法即可。首先我们传递选取所有商品对应的 XPath，可以匹配所有商品，随后对结果进行遍历，依次选取每个商品的名称、价格、图片等内容，构造并返回一个 ProductItem 对象。</p>
                  <h3 id="7-存储结果"><a href="#7-存储结果" class="headerlink" title="7. 存储结果"></a>7. 存储结果</h3>
                  <p>最后我们实现一个 Item Pipeline，将结果保存到 MongoDB，如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymongo</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>), mongo_db=crawler.settings.get(<span class="string">'MONGO_DB'</span>))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db[item.collection].insert(dict(item))</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此实现和前文中存储到 MongoDB 的方法完全一致，原理不再赘述。记得在 settings.py 中开启它的调用，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">ITEM_PIPELINES</span> = &#123;<span class="string">'scrapyseleniumtest.pipelines.MongoPipeline'</span>: <span class="number">300</span>,&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中，MONGO_URI 和 MONGO_DB 的定义如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MONGO_URI</span> = <span class="string">'localhost'</span></span><br><span class="line"><span class="attr">MONGO_DB</span> = <span class="string">'taobao'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="8-运行"><a href="#8-运行" class="headerlink" title="8. 运行"></a>8. 运行</h3>
                  <p>整个项目就完成了，执行如下命令启动抓取即可：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl taobao</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图 13-13 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034100.jpg" alt=""> 图 13-13 运行结果 再查看一下 MongoDB，结果如图 13-14 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-034105.jpg" alt=""> 图 13-14 MongoDB 结果 这样我们便成功在 Scrapy 中对接 Selenium 并实现了淘宝商品的抓取。</p>
                  <h3 id="9-本节代码"><a href="#9-本节代码" class="headerlink" title="9. 本节代码"></a>9. 本节代码</h3>
                  <p>本节代码地址为：<a href="https://github.com/Python3WebSpider/ScrapySeleniumTest" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapySeleniumTest</a>。</p>
                  <h3 id="10-结语"><a href="#10-结语" class="headerlink" title="10. 结语"></a>10. 结语</h3>
                  <p>我们通过改写 Downloader Middleware 的方式实现了 Selenium 的对接。但这种方法其实是阻塞式的，也就是说这样就破坏了 Scrapy 异步处理的逻辑，速度会受到影响。为了不破坏其异步加载逻辑，我们可以使用 Splash 实现。下一节我们再来看看 Scrapy 对接 Splash 的方式。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-05 09:30:23" itemprop="dateCreated datePublished" datetime="2019-12-05T09:30:23+08:00">2019-12-05</time>
                </span>
                <span id="/8397.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.8–Scrapy 对接 Selenium" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8394.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8394.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.7–Item Pipeline 的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-7-Item-Pipeline-的用法"><a href="#13-7-Item-Pipeline-的用法" class="headerlink" title="13.7 Item Pipeline 的用法"></a>13.7 Item Pipeline 的用法</h1>
                  <p>Item Pipeline 是项目管道。在前面我们已经了解了 Item Pipeline 的基本用法，本节我们再作详细了解它的用法。 首先我们看看 Item Pipeline 在 Scrapy 中的架构，如图 13-1 所示。 图中的最左侧即为 Item Pipeline，它的调用发生在 Spider 产生 Item 之后。当 Spider 解析完 Response 之后，Item 就会传递到 Item Pipeline，被定义的 Item Pipeline 组件会顺次调用，完成一连串的处理过程，比如数据清洗、存储等。 它的主要功能有：</p>
                  <ul>
                    <li>清洗 HTML 数据</li>
                    <li>验证爬取数据，检查爬取字段</li>
                    <li>查重并丢弃重复内容</li>
                    <li>将爬取结果储存到数据库</li>
                  </ul>
                  <h3 id="1-核心方法"><a href="#1-核心方法" class="headerlink" title="1. 核心方法"></a>1. 核心方法</h3>
                  <p>我们可以自定义 Item Pipeline，只需要实现指定的方法就好，其中必须要实现的一个方法是：</p>
                  <ul>
                    <li>process_item(item, spider)</li>
                  </ul>
                  <p>另外还有几个比较实用的方法，它们分别是：</p>
                  <ul>
                    <li>open_spider(spider)</li>
                    <li>close_spider(spider)</li>
                    <li>from_crawler(cls, crawler)</li>
                  </ul>
                  <p>下面我们对这几个方法的用法作下详细的介绍：</p>
                  <h4 id="process-item-item-spider"><a href="#process-item-item-spider" class="headerlink" title="process_item(item, spider)"></a>process_item(item, spider)</h4>
                  <p>process_item() 是必须要实现的方法，被定义的 Item Pipeline 会默认调用这个方法对 Item 进行处理。比如，我们可以进行数据处理或者将数据写入到数据库等操作。它必须返回 Item 类型的值或者抛出一个 DropItem 异常。 process_item() 方法的参数有如下两个。</p>
                  <ul>
                    <li>item，是 Item 对象，即被处理的 Item</li>
                    <li>spider，是 Spider 对象，即生成该 Item 的 Spider</li>
                  </ul>
                  <p>下面对该方法的返回类型归纳如下：</p>
                  <ul>
                    <li>如果返回的是 Item 对象，那么此 Item 会接着被低优先级的 Item Pipeline 的 process_item() 方法进行处理，直到所有的方法被调用完毕。</li>
                    <li>如果抛出的是 DropItem 异常，那么此 Item 就会被丢弃，不再进行处理。</li>
                  </ul>
                  <h4 id="open-spider-self-spider"><a href="#open-spider-self-spider" class="headerlink" title="open_spider(self, spider)"></a>open_spider(self, spider)</h4>
                  <p>open_spider() 方法是在 Spider 开启的时候被自动调用的，在这里我们可以做一些初始化操作，如开启数据库连接等。其中参数 spider 就是被开启的 Spider 对象。</p>
                  <h4 id="close-spider-spider"><a href="#close-spider-spider" class="headerlink" title="close_spider(spider)"></a>close_spider(spider)</h4>
                  <p>close_spider() 方法是在 Spider 关闭的时候自动调用的，在这里我们可以做一些收尾工作，如关闭数据库连接等，其中参数 spider 就是被关闭的 Spider 对象。</p>
                  <h4 id="from-crawler-cls-crawler"><a href="#from-crawler-cls-crawler" class="headerlink" title="from_crawler(cls, crawler)"></a>from_crawler(cls, crawler)</h4>
                  <p>from_crawler() 方法是一个类方法，用 @classmethod 标识，是一种依赖注入的方式。它的参数是 crawler，通过 crawler 对象，我们可以拿到 Scrapy 的所有核心组件，如全局配置的每个信息，然后创建一个 Pipeline 实例。参数 cls 就是 Class，最后返回一个 Class 实例。 下面我们用一个实例来加深对 Item Pipeline 用法的理解。</p>
                  <h3 id="2-本节目标"><a href="#2-本节目标" class="headerlink" title="2. 本节目标"></a>2. 本节目标</h3>
                  <p>我们以爬取 360 摄影美图为例，来分别实现 MongoDB 存储、MySQL 存储、Image 图片存储的三个 Pipeline。</p>
                  <h3 id="3-准备工作"><a href="#3-准备工作" class="headerlink" title="3. 准备工作"></a>3. 准备工作</h3>
                  <p>请确保已经安装好 MongoDB 和 MySQL 数据库，安装好 Python 的 PyMongo、PyMySQL、Scrapy 框架，另外需要安装 pillow 图像处理库，如没有安装可以参考第 1 章的安装说明。</p>
                  <h3 id="4-抓取分析"><a href="#4-抓取分析" class="headerlink" title="4. 抓取分析"></a>4. 抓取分析</h3>
                  <p>我们这次爬取的目标网站为：<a href="https://image.so.com。打开此页面，切换到摄影页面，网页中呈现了许许多多的摄影美图。我们打开浏览器开发者工具，过滤器切换到" target="_blank" rel="noopener">https://image.so.com。打开此页面，切换到摄影页面，网页中呈现了许许多多的摄影美图。我们打开浏览器开发者工具，过滤器切换到</a> XHR 选项，然后下拉页面，可以看到下面就会呈现许多 Ajax 请求，如图 13-6 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-033958.png" alt=""> 图 13-6 请求列表 我们查看一个请求的详情，观察返回的数据结构，如图 13-7 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034003.jpg" alt=""> 图 13-7 返回结果 返回格式是 JSON。其中 list 字段就是一张张图片的详情信息，包含了 30 张图片的 ID、名称、链接、缩略图等信息。另外观察 Ajax 请求的参数信息，有一个参数 sn 一直在变化，这个参数很明显就是偏移量。当 sn 为 30 时，返回的是前 30 张图片，sn 为 60 时，返回的就是第 31~60 张图片。另外，ch 参数是摄影类别，listtype 是排序方式，temp 参数可以忽略。 所以我们抓取时只需要改变 sn 的数值就好了。 下面我们用 Scrapy 来实现图片的抓取，将图片的信息保存到 MongoDB、MySQL，同时将图片存储到本地。</p>
                  <h3 id="5-新建项目"><a href="#5-新建项目" class="headerlink" title="5. 新建项目"></a>5. 新建项目</h3>
                  <p>首先新建一个项目，命令如下：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">scrapy </span>startproject images360</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来新建一个 Spider，命令如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">images</span> <span class="selector-tag">images</span><span class="selector-class">.so</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功创建了一个 Spider。</p>
                  <h3 id="6-构造请求"><a href="#6-构造请求" class="headerlink" title="6. 构造请求"></a>6. 构造请求</h3>
                  <p>接下来定义爬取的页数。比如爬取 50 页、每页 30 张，也就是 1500 张图片，我们可以先在 settings.py 里面定义一个变量 MAX_PAGE，添加如下定义：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MAX_PAGE</span> = <span class="number">50</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>定义 start_requests() 方法，用来生成 50 次请求，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def start_requests(self):</span><br><span class="line">    data = &#123;<span class="string">'ch'</span>: <span class="string">'photography'</span>, <span class="string">'listtype'</span>: <span class="string">'new'</span>&#125;</span><br><span class="line">    base_url = <span class="string">'https://image.so.com/zj?'</span></span><br><span class="line">    <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, self.settings.<span class="builtin-name">get</span>(<span class="string">'MAX_PAGE'</span>) + 1):</span><br><span class="line">        data[<span class="string">'sn'</span>] =<span class="built_in"> page </span>* 30</span><br><span class="line">        params = urlencode(data)</span><br><span class="line">        url = base_url + params</span><br><span class="line">        yield Request(url, self.parse)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们首先定义了初始的两个参数，sn 参数是遍历循环生成的。然后利用 urlencode() 方法将字典转化为 URL 的 GET 参数，构造出完整的 URL，构造并生成 Request。 还需要引入 scrapy.Request 和 urllib.parse 模块，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Spider, Request</span><br><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlencode</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再修改 settings.py 中的 ROBOTSTXT_OBEY 变量，将其设置为 False，否则无法抓取，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">ROBOTSTXT_OBEY</span> = <span class="literal">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行爬虫，即可以看到链接都请求成功，执行命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl images</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行示例结果如图 13-8 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034009.jpg" alt=""> 图 13-8 运行结果 所有请求的状态码都是 200，这就证明图片信息爬取成功了。</p>
                  <h3 id="7-提取信息"><a href="#7-提取信息" class="headerlink" title="7. 提取信息"></a>7. 提取信息</h3>
                  <p>首先定义一个 Item，叫作 ImageItem，如下所示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Item, Field</span><br><span class="line"><span class="keyword">class</span> <span class="symbol">ImageItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = <span class="symbol">table</span> = '<span class="symbol">images</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">url</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">title</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">thumb</span> = <span class="symbol">Field</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们定义了 4 个字段，包括图片的 ID、链接、标题、缩略图。另外还有两个属性 collection 和 table，都定义为 images 字符串，分别代表 MongoDB 存储的 Collection 名称和 MySQL 存储的表名称。 接下来我们提取 Spider 里有关信息，将 parse() 方法改写为如下所示：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse(self, response):</span><br><span class="line">    <span class="built_in">result</span> = json.loads(response.<span class="keyword">text</span>)</span><br><span class="line">    <span class="keyword">for</span> image <span class="keyword">in</span> <span class="built_in">result</span>.<span class="built_in">get</span>(<span class="string">'list'</span>):</span><br><span class="line">        <span class="keyword">item</span> = ImageItem()</span><br><span class="line">        <span class="keyword">item</span>[<span class="string">'id'</span>] = image.<span class="built_in">get</span>(<span class="string">'imageid'</span>)</span><br><span class="line">        <span class="keyword">item</span>[<span class="string">'url'</span>] = image.<span class="built_in">get</span>(<span class="string">'qhimg_url'</span>)</span><br><span class="line">        <span class="keyword">item</span>[<span class="string">'title'</span>] = image.<span class="built_in">get</span>(<span class="string">'group_title'</span>)</span><br><span class="line">        <span class="keyword">item</span>[<span class="string">'thumb'</span>] = image.<span class="built_in">get</span>(<span class="string">'qhimg_thumb_url'</span>)</span><br><span class="line">        yield <span class="keyword">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先解析 JSON，遍历其 list 字段，取出一个个图片信息，然后再对 ImageItem 赋值，生成 Item 对象。 这样我们就完成了信息的提取。</p>
                  <h3 id="8-存储信息"><a href="#8-存储信息" class="headerlink" title="8. 存储信息"></a>8. 存储信息</h3>
                  <p>接下来我们需要将图片的信息保存到 MongoDB、MySQL，同时将图片保存到本地。</p>
                  <h4 id="MongoDB"><a href="#MongoDB" class="headerlink" title="MongoDB"></a>MongoDB</h4>
                  <p>首先确保 MongoDB 已经正常安装并且正常运行。 我们用一个 MongoPipeline 将信息保存到 MongoDB，在 pipelines.py 里添加如下类的实现：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymongo</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>),</span><br><span class="line">            mongo_db=crawler.settings.get(<span class="string">'MONGO_DB'</span>)</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db[item.collection].insert(dict(item))</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要用到两个变量，MONGO_URI 和 MONGO_DB，即存储到 MongoDB 的链接地址和数据库名称。我们在 settings.py 里添加这两个变量，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MONGO_URI</span> = <span class="string">'localhost'</span></span><br><span class="line"><span class="attr">MONGO_DB</span> = <span class="string">'images360'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样一个保存到 MongoDB 的 Pipeline 的就创建好了。这里最主要的方法是 process_item() 方法，直接调用 Collection 对象的 insert() 方法即可完成数据的插入，最后返回 Item 对象。</p>
                  <h4 id="MySQL"><a href="#MySQL" class="headerlink" title="MySQL"></a>MySQL</h4>
                  <p>首先确保 MySQL 已经正确安装并且正常运行。 新建一个数据库，名字还是 images360，SQL 语句如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">CREATE DATABASE images360<span class="built_in"> DEFAULT </span>CHARACTER <span class="builtin-name">SET</span> utf8 COLLATE utf8_general_ci</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建一个数据表，包含 id、url、title、thumb 四个字段，SQL 语句如下所示：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> images (<span class="keyword">id</span> <span class="built_in">VARCHAR</span>(<span class="number">255</span>) <span class="literal">NULL</span> PRIMARY <span class="keyword">KEY</span>, <span class="keyword">url</span> <span class="built_in">VARCHAR</span>(<span class="number">255</span>) <span class="literal">NULL</span> , title <span class="built_in">VARCHAR</span>(<span class="number">255</span>) <span class="literal">NULL</span> , thumb <span class="built_in">VARCHAR</span>(<span class="number">255</span>) <span class="literal">NULL</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完 SQL 语句之后，我们就成功创建好了数据表。接下来就可以往表里存储数据了。 接下来我们实现一个 MySQLPipeline，代码如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymysql</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MysqlPipeline</span>():</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, host, database, user, password, port)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.host = host</span><br><span class="line">        <span class="keyword">self</span>.database = database</span><br><span class="line">        <span class="keyword">self</span>.user = user</span><br><span class="line">        <span class="keyword">self</span>.password = password</span><br><span class="line">        <span class="keyword">self</span>.port = port</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(host=crawler.settings.get(<span class="string">'MYSQL_HOST'</span>),</span><br><span class="line">            database=crawler.settings.get(<span class="string">'MYSQL_DATABASE'</span>),</span><br><span class="line">            user=crawler.settings.get(<span class="string">'MYSQL_USER'</span>),</span><br><span class="line">            password=crawler.settings.get(<span class="string">'MYSQL_PASSWORD'</span>),</span><br><span class="line">            port=crawler.settings.get(<span class="string">'MYSQL_PORT'</span>),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db = pymysql.connect(<span class="keyword">self</span>.host, <span class="keyword">self</span>.user, <span class="keyword">self</span>.password, <span class="keyword">self</span>.database, charset=<span class="string">'utf8'</span>, port=<span class="keyword">self</span>.port)</span><br><span class="line">        <span class="keyword">self</span>.cursor = <span class="keyword">self</span>.db.cursor()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        data = dict(item)</span><br><span class="line">        keys = <span class="string">', '</span>.join(data.keys())</span><br><span class="line">        values = <span class="string">', '</span>.join([<span class="string">'% s'</span>] * len(data))</span><br><span class="line">        sql = <span class="string">'insert into % s (% s) values (% s)'</span> % (item.table, keys, values)</span><br><span class="line">        <span class="keyword">self</span>.cursor.execute(sql, tuple(data.values()))</span><br><span class="line">        <span class="keyword">self</span>.db.commit()</span><br><span class="line">        <span class="keyword">return</span> item</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如前所述，这里用到的数据插入方法是一个动态构造 SQL 语句的方法。 这里又需要几个 MySQL 的配置，我们在 settings.py 里添加几个变量，如下所示：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">MYSQL_HOST</span> = <span class="string">'localhost'</span></span><br><span class="line"><span class="attr">MYSQL_DATABASE</span> = <span class="string">'images360'</span></span><br><span class="line"><span class="attr">MYSQL_PORT</span> = <span class="number">3306</span></span><br><span class="line"><span class="attr">MYSQL_USER</span> = <span class="string">'root'</span></span><br><span class="line"><span class="attr">MYSQL_PASSWORD</span> = <span class="string">'123456'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里分别定义了 MySQL 的地址、数据库名称、端口、用户名、密码。 这样，MySQL Pipeline 就完成了。</p>
                  <h4 id="Image-Pipeline"><a href="#Image-Pipeline" class="headerlink" title="Image Pipeline"></a>Image Pipeline</h4>
                  <p>Scrapy 提供了专门处理下载的 Pipeline，包括文件下载和图片下载。下载文件和图片的原理与抓取页面的原理一样，因此下载过程支持异步和多线程，下载十分高效。下面我们来看看具体的实现过程。 官方文档地址为：<a href="https://doc.scrapy.org/en/latest/topics/media-pipeline.html" target="_blank" rel="noopener">https://doc.scrapy.org/en/latest/topics/media-pipeline.html</a>。 首先定义存储文件的路径，需要定义一个 IMAGES_STORE 变量，在 settings.py 中添加如下代码：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">IMAGES_STORE</span> = <span class="string">'./images'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们将路径定义为当前路径下的 images 子文件夹，即下载的图片都会保存到本项目的 images 文件夹中。 内置的 ImagesPipeline 会默认读取 Item 的 image_urls 字段，并认为该字段是一个列表形式，它会遍历 Item 的 image_urls 字段，然后取出每个 URL 进行图片下载。 但是现在生成的 Item 的图片链接字段并不是 image_urls 字段表示的，也不是列表形式，而是单个的 URL。所以为了实现下载，我们需要重新定义下载的部分逻辑，即要自定义 ImagePipeline，继承内置的 ImagesPipeline，重写几个方法。 我们定义 ImagePipeline，如下所示：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Request</span><br><span class="line"><span class="keyword">from</span> scrapy.exceptions <span class="keyword">import</span> DropItem</span><br><span class="line"><span class="keyword">from</span> scrapy.pipelines.images <span class="keyword">import</span> ImagesPipeline</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ImagePipeline</span><span class="params">(ImagesPipeline)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">file_path</span><span class="params">(self, request, response=None, info=None)</span>:</span></span><br><span class="line">        url = request.url</span><br><span class="line">        file_name = url.split(<span class="string">'/'</span>)[<span class="number">-1</span>]</span><br><span class="line">        <span class="keyword">return</span> file_name</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">item_completed</span><span class="params">(self, results, item, info)</span>:</span></span><br><span class="line">        image_paths = [x[<span class="string">'path'</span>] <span class="keyword">for</span> ok, x <span class="keyword">in</span> results <span class="keyword">if</span> ok]</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> image_paths:</span><br><span class="line">            <span class="keyword">raise</span> DropItem(<span class="string">'Image Downloaded Failed'</span>)</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_media_requests</span><span class="params">(self, item, info)</span>:</span></span><br><span class="line">        <span class="keyword">yield</span> Request(item[<span class="string">'url'</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们实现了 ImagePipeline，继承 Scrapy 内置的 ImagesPipeline，重写下面几个方法。</p>
                  <ul>
                    <li>get_media_requests()。它的第一个参数 item 是爬取生成的 Item 对象。我们将它的 url 字段取出来，然后直接生成 Request 对象。此 Request 加入到调度队列，等待被调度，执行下载。</li>
                    <li>file_path()。它的第一个参数 request 就是当前下载对应的 Request 对象。这个方法用来返回保存的文件名，直接将图片链接的最后一部分当作文件名即可。它利用 split() 函数分割链接并提取最后一部分，返回结果。这样此图片下载之后保存的名称就是该函数返回的文件名。</li>
                    <li>item_completed()，它是当单个 Item 完成下载时的处理方法。因为并不是每张图片都会下载成功，所以我们需要分析下载结果并剔除下载失败的图片。如果某张图片下载失败，那么我们就不需保存此 Item 到数据库。该方法的第一个参数 results 就是该 Item 对应的下载结果，它是一个列表形式，列表每一个元素是一个元组，其中包含了下载成功或失败的信息。这里我们遍历下载结果找出所有成功的下载列表。如果列表为空，那么该 Item 对应的图片下载失败，随即抛出异常 DropItem，该 Item 忽略。否则返回该 Item，说明此 Item 有效。</li>
                  </ul>
                  <p>现在为止，三个 Item Pipeline 的定义就完成了。最后只需要启用就可以了，修改 settings.py，设置 ITEM_PIPELINES，如下所示：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">ITEM_PIPELINES</span> <span class="string">=</span> <span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'images360.pipelines.ImagePipeline':</span> <span class="number">300</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'images360.pipelines.MongoPipeline':</span> <span class="number">301</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'images360.pipelines.MysqlPipeline':</span> <span class="number">302</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里注意调用的顺序。我们需要优先调用 ImagePipeline 对 Item 做下载后的筛选，下载失败的 Item 就直接忽略，它们就不会保存到 MongoDB 和 MySQL 里。随后再调用其他两个存储的 Pipeline，这样就能确保存入数据库的图片都是下载成功的。 接下来运行程序，执行爬取，如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl images</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>爬虫一边爬取一边下载，下载速度非常快，对应的输出日志如图 13-9 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034019.jpg" alt=""> 图 13-9 输出日志 查看本地 images 文件夹，发现图片都已经成功下载，如图 13-10 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034024.jpg" alt=""> 图 13-10 下载结果 查看 MySQL，下载成功的图片信息也已成功保存，如图 13-11 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034029.jpg" alt=""> 图 13-11 MySQL 结果 查看 MongoDB，下载成功的图片信息同样已成功保存，如图 13-12 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034034.jpg" alt=""> 图 13-12 MongoDB 结果 这样我们就可以成功实现图片的下载并把图片的信息存入数据库了。</p>
                  <h3 id="9-本节代码"><a href="#9-本节代码" class="headerlink" title="9. 本节代码"></a>9. 本节代码</h3>
                  <p>本节代码地址为：<a href="https://github.com/Python3WebSpider/Images360" target="_blank" rel="noopener">https://github.com/Python3WebSpider/Images360</a>。</p>
                  <h3 id="10-结语"><a href="#10-结语" class="headerlink" title="10. 结语"></a>10. 结语</h3>
                  <p>Item Pipeline 是 Scrapy 非常重要的组件，数据存储几乎都是通过此组件实现的。请读者认真掌握此内容。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-05 09:26:32" itemprop="dateCreated datePublished" datetime="2019-12-05T09:26:32+08:00">2019-12-05</time>
                </span>
                <span id="/8394.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.7–Item Pipeline 的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8385.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8385.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.6–Spider Middleware 的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-6-Spider-Middleware-的用法"><a href="#13-6-Spider-Middleware-的用法" class="headerlink" title="13.6 Spider Middleware 的用法"></a>13.6 Spider Middleware 的用法</h1>
                  <p>Spider Middleware 是介入到 Scrapy 的 Spider 处理机制的钩子框架。我们首先来看看它的架构，如图 13-1 所示。 当 Downloader 生成 Response 之后，Response 会被发送给 Spider，在发送给 Spider 之前，Response 会首先经过 Spider Middleware 处理，当 Spider 处理生成 Item 和 Request 之后，Item 和 Request 还会经过 Spider Middleware 的处理。 Spider Middleware 有如下三个作用。</p>
                  <ul>
                    <li>我们可以在 Downloader 生成的 Response 发送给 Spider 之前，也就是在 Response 发送给 Spider 之前对 Response 进行处理。</li>
                    <li>我们可以在 Spider 生成的 Request 发送给 Scheduler 之前，也就是在 Request 发送给 Scheduler 之前对 Request 进行处理。</li>
                    <li>我们可以在 Spider 生成的 Item 发送给 Item Pipeline 之前，也就是在 Item 发送给 Item Pipeline 之前对 Item 进行处理。</li>
                  </ul>
                  <h3 id="1-使用说明"><a href="#1-使用说明" class="headerlink" title="1. 使用说明"></a>1. 使用说明</h3>
                  <p>需要说明的是，Scrapy 其实已经提供了许多 Spider Middleware，它们被 SPIDER_MIDDLEWARES_BASE 这个变量所定义。 SPIDER_MIDDLEWARES_BASE 变量的内容如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware':</span> <span class="number">50</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.spidermiddlewares.offsite.OffsiteMiddleware':</span> <span class="number">500</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.spidermiddlewares.referer.RefererMiddleware':</span> <span class="number">700</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware':</span> <span class="number">800</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.spidermiddlewares.depth.DepthMiddleware':</span> <span class="number">900</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>和 Downloader Middleware 一样，Spider Middleware 首先加入到 SPIDER_MIDDLEWARES 设置中，该设置会和 Scrapy 中 SPIDER_MIDDLEWARES_BASE 定义的 Spider Middleware 合并。然后根据键值的数字优先级排序，得到一个有序列表。第一个 Middleware 是最靠近引擎的，最后一个 Middleware 是最靠近 Spider 的。</p>
                  <h3 id="2-核心方法"><a href="#2-核心方法" class="headerlink" title="2. 核心方法"></a>2. 核心方法</h3>
                  <p>Scrapy 内置的 Spider Middleware 为 Scrapy 提供了基础的功能。如果我们想要扩展其功能，只需要实现某几个方法即可。 每个 Spider Middleware 都定义了以下一个或多个方法的类，核心方法有如下 4 个。</p>
                  <ul>
                    <li>process_spider_input(response, spider)</li>
                    <li>process_spider_output(response, result, spider)</li>
                    <li>process_spider_exception(response, exception, spider)</li>
                    <li>process_start_requests(start_requests, spider)</li>
                  </ul>
                  <p>只需要实现其中一个方法就可以定义一个 Spider Middleware。下面我们来看看这 4 个方法的详细用法。</p>
                  <h4 id="process-spider-input-response-spider"><a href="#process-spider-input-response-spider" class="headerlink" title="process_spider_input(response, spider)"></a>process_spider_input(response, spider)</h4>
                  <p>当 Response 通过 Spider Middleware 时，该方法被调用，处理该 Response。 方法的参数有两个：</p>
                  <ul>
                    <li>response，即 Response 对象，即被处理的 Response</li>
                    <li>spider，即 Spider 对象，即该 response 对应的 Spider</li>
                  </ul>
                  <p>process_spider_input() 应该返回 None 或者抛出一个异常。</p>
                  <ul>
                    <li>如果其返回 None ，Scrapy 将会继续处理该 Response，调用所有其他的 Spider Middleware 直到 Spider 处理该 Response。</li>
                    <li>如果其抛出一个异常，Scrapy 将不会调用任何其他 Spider Middlewar e 的 process_spider_input() 方法，并调用 Request 的 errback() 方法。 errback 的输出将会以另一个方向被重新输入到中间件中，使用 process_spider_output() 方法来处理，当其抛出异常时则调用 process_spider_exception() 来处理。</li>
                  </ul>
                  <h4 id="process-spider-output-response-result-spider"><a href="#process-spider-output-response-result-spider" class="headerlink" title="process_spider_output(response, result, spider)"></a>process_spider_output(response, result, spider)</h4>
                  <p>当 Spider 处理 Response 返回结果时，该方法被调用。 方法的参数有三个：</p>
                  <ul>
                    <li>response，即 Response 对象，即生成该输出的 Response</li>
                    <li>result，包含 Request 或 Item 对象的可迭代对象，即 Spider 返回的结果</li>
                    <li>spider，即 Spider 对象，即其结果对应的 Spider</li>
                  </ul>
                  <p>process_spider_output() 必须返回包含 Request 或 Item 对象的可迭代对象。</p>
                  <h4 id="process-spider-exception-response-exception-spider"><a href="#process-spider-exception-response-exception-spider" class="headerlink" title="process_spider_exception(response, exception, spider)"></a>process_spider_exception(response, exception, spider)</h4>
                  <p>当 Spider 或 Spider Middleware 的 process_spider_input() 方法抛出异常时， 该方法被调用。 方法的参数有三个：</p>
                  <ul>
                    <li>response，即 Response 对象，即异常被抛出时被处理的 Response</li>
                    <li>exception，即 Exception 对象，被抛出的异常</li>
                    <li>spider，即 Spider 对象，即抛出该异常的 Spider</li>
                  </ul>
                  <p>process_spider_exception() 必须要么返回 None ， 要么返回一个包含 Response 或 Item 对象的可迭代对象。</p>
                  <ul>
                    <li>如果其返回 None ，Scrapy 将继续处理该异常，调用其他 Spider Middleware 中的 process_spider_exception() 方法，直到所有 Spider Middleware 都被调用。</li>
                    <li>如果其返回一个可迭代对象，则其他 Spider Middleware 的 process_spider_output() 方法被调用， 其他的 process_spider_exception() 将不会被调用。</li>
                  </ul>
                  <h4 id="process-start-requests-start-requests-spider"><a href="#process-start-requests-start-requests-spider" class="headerlink" title="process_start_requests(start_requests, spider)"></a>process_start_requests(start_requests, spider)</h4>
                  <p>该方法以 Spider 启动的 Request 为参数被调用，执行的过程类似于 process_spider_output() ，只不过其没有相关联的 Response 并且必须返回 Request。 方法的参数有两个：</p>
                  <ul>
                    <li>start_requests，即包含 Request 的可迭代对象，即 Start Requests</li>
                    <li>spider，即 Spider 对象，即 Start Requests 所属的 Spider</li>
                  </ul>
                  <p>其必须返回另一个包含 Request 对象的可迭代对象。</p>
                  <h3 id="3-结语"><a href="#3-结语" class="headerlink" title="3. 结语"></a>3. 结语</h3>
                  <p>本节介绍了 Spider Middleware 的基本原理和自定义 Spider Middleware 的方法。Spider Middleware 使用的频率不如 Downloader Middleware 的高，在必要的情况下它可以用来方便数据的处理。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-04 15:18:30" itemprop="dateCreated datePublished" datetime="2019-12-04T15:18:30+08:00">2019-12-04</time>
                </span>
                <span id="/8385.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.6–Spider Middleware 的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8381.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8381.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.5–Downloader Middleware 的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-5-Downloader-Middleware-的用法"><a href="#13-5-Downloader-Middleware-的用法" class="headerlink" title="13.5 Downloader Middleware 的用法"></a>13.5 Downloader Middleware 的用法</h1>
                  <p>Downloader Middleware 即下载中间件，它是处于 Scrapy 的 Request 和 Response 之间的处理模块。我们首先来看看它的架构，如图 13-1 所示。 Scheduler 从队列中拿出一个 Request 发送给 Downloader 执行下载，这个过程会经过 Downloader Middleware 的处理。另外，当 Downloader 将 Request 下载完成得到 Response 返回给 Spider 时会再次经过 Downloader Middleware 处理。 也就是说，Downloader Middleware 在整个架构中起作用的位置是以下两个。</p>
                  <ul>
                    <li>在 Scheduler 调度出队列的 Request 发送给 Downloader 下载之前，也就是我们可以在 Request 执行下载之前对其进行修改。</li>
                    <li>在下载后生成的 Response 发送给 Spider 之前，也就是我们可以在生成 Resposne 被 Spider 解析之前对其进行修改。</li>
                  </ul>
                  <p>Downloader Middleware 的功能十分强大，修改 User-Agent、处理重定向、设置代理、失败重试、设置 Cookies 等功能都需要借助它来实现。下面我们来了解一下 Downloader Middleware 的详细用法。</p>
                  <h3 id="1-使用说明"><a href="#1-使用说明" class="headerlink" title="1. 使用说明"></a>1. 使用说明</h3>
                  <p>需要说明的是，Scrapy 其实已经提供了许多 Downloader Middleware，比如负责失败重试、自动重定向等功能的 Middleware，它们被 DOWNLOADER_MIDDLEWARES_BASE 变量所定义。 DOWNLOADER_MIDDLEWARES_BASE 变量的内容如下所示：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware':</span> <span class="number">100</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware':</span> <span class="number">300</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':</span> <span class="number">350</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':</span> <span class="number">400</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':</span> <span class="number">500</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.retry.RetryMiddleware':</span> <span class="number">550</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware':</span> <span class="number">560</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware':</span> <span class="number">580</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':</span> <span class="number">590</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.RedirectMiddleware':</span> <span class="number">600</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':</span> <span class="number">700</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':</span> <span class="number">750</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.stats.DownloaderStats':</span> <span class="number">850</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware':</span> <span class="number">900</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是一个字典格式，字典的键名是 Scrapy 内置的 Downloader Middleware 的名称，键值代表了调用的优先级，优先级是一个数字，数字越小代表越靠近 Scrapy 引擎，数字越大代表越靠近 Downloader。每个 Downloader Middleware 都可以定义 process_request() 和 request_response() 方法来分别处理请求和响应，对于 process_request() 方法来说，优先级数字越小越先被调用，对于 process_response() 方法来说，优先级数字越大越先被调用。。 如果自己定义的 Downloader Middleware 要添加到项目里，DOWNLOADER_MIDDLEWARES_BASE 变量不能直接修改。Scrapy 提供了另外一个设置变量 DOWNLOADER_MIDDLEWARES，我们直接修改这个变量就可以添加自己定义的 Downloader Middleware，以及禁用 DOWNLOADER_MIDDLEWARES_BASE 里面定义的 Downloader Middleware。下面我们具体来看看 Downloader Middleware 的使用方法。</p>
                  <h3 id="2-核心方法"><a href="#2-核心方法" class="headerlink" title="2. 核心方法"></a>2. 核心方法</h3>
                  <p>Scrapy 内置的 Downloader Middleware 为 Scrapy 提供了基础的功能，但在项目实战中我们往往需要单独定义 Downloader Middleware。不用担心，这个过程非常简单，我们只需要实现某几个方法即可。 每个 Downloader Middleware 都定义了一个或多个方法的类，核心的方法有如下三个。</p>
                  <ul>
                    <li>process_request(request, spider)</li>
                    <li>process_response(request, response, spider)</li>
                    <li>process_exception(request, exception, spider)</li>
                  </ul>
                  <p>我们只需要实现至少一个方法，就可以定义一个 Downloader Middleware。下面我们来看看这三个方法的详细用法。</p>
                  <h4 id="process-request-request-spider"><a href="#process-request-request-spider" class="headerlink" title="process_request(request, spider)"></a>process_request(request, spider)</h4>
                  <p>Request 被 Scrapy 引擎调度给 Downloader 之前，process_request() 方法就会被调用，也就是在 Request 从队列里调度出来到 Downloader 下载执行之前，我们都可以用 process_request() 方法对 Request 进行处理。方法的返回值必须为 None、Response 对象、Request 对象之一，或者抛出 IgnoreRequest 异常。 process_request() 方法的参数有如下两个。</p>
                  <ul>
                    <li>request，即 Request 对象，即被处理的 Request</li>
                    <li>spider，即 Spdier 对象，即此 Request 对应的 Spider</li>
                  </ul>
                  <p>返回类型不同，产生的效果也不同。下面归纳一下不同的返回情况。</p>
                  <ul>
                    <li>当返回是 None 时，Scrapy 将继续处理该 Request，接着执行其他 Downloader Middleware 的 process_request() 方法，一直到 Downloader 把 Request 执行后得到 Response 才结束。这个过程其实就是修改 Request 的过程，不同的 Downloader Middleware 按照设置的优先级顺序依次对 Request 进行修改，最后送至 Downloader 执行。</li>
                    <li>当返回为 Response 对象时，更低优先级的 Downloader Middleware 的 process_request() 和 process_exception() 方法就不会被继续调用，每个 Downloader Middleware 的 process_response() 方法转而被依次调用。调用完毕之后，直接将 Response 对象发送给 Spider 来处理。</li>
                    <li>当返回为 Request 对象时，更低优先级的 Downloader Middleware 的 process_request() 方法会停止执行。这个 Request 会重新放到调度队列里，其实它就是一个全新的 Request，等待被调度。如果被 Scheduler 调度了，那么所有的 Downloader Middleware 的 process_request() 方法会被重新按照顺序执行。</li>
                    <li>如果 IgnoreRequest 异常抛出，则所有的 Downloader Middleware 的 process_exception() 方法会依次执行。如果没有一个方法处理这个异常，那么 Request 的 errorback() 方法就会回调。如果该异常还没有被处理，那么它便会被忽略。</li>
                  </ul>
                  <h4 id="process-response-request-response-spider"><a href="#process-response-request-response-spider" class="headerlink" title="process_response(request, response, spider)"></a>process_response(request, response, spider)</h4>
                  <p>Downloader 执行 Request 下载之后，会得到对应的 Response。Scrapy 引擎便会将 Response 发送给 Spider 进行解析。在发送之前，我们都可以用 process_response() 方法来对 Response 进行处理。方法的返回值必须为 Request 对象、Response 对象之一，或者抛出 IgnoreRequest 异常。 process_response() 方法的参数有如下三个。</p>
                  <ul>
                    <li>request，是 Request 对象，即此 Response 对应的 Request。</li>
                    <li>response，是 Response 对象，即此被处理的 Response。</li>
                    <li>spider，是 Spider 对象，即此 Response 对应的 Spider。</li>
                  </ul>
                  <p>下面对不同的返回情况做一下归纳：</p>
                  <ul>
                    <li>当返回为 Request 对象时，更低优先级的 Downloader Middleware 的 process_response() 方法不会继续调用。该 Request 对象会重新放到调度队列里等待被调度，它相当于一个全新的 Request。然后，该 Request 会被 process_request() 方法顺次处理。</li>
                    <li>当返回为 Response 对象时，更低优先级的 Downloader Middleware 的 process_response() 方法会继续调用，继续对该 Response 对象进行处理。</li>
                    <li>如果 IgnoreRequest 异常抛出，则 Request 的 errorback() 方法会回调。如果该异常还没有被处理，那么它便会被忽略。</li>
                  </ul>
                  <h4 id="process-exception-request-exception-spider"><a href="#process-exception-request-exception-spider" class="headerlink" title="process_exception(request, exception, spider)"></a>process_exception(request, exception, spider)</h4>
                  <p>当 Downloader 或 process_request() 方法抛出异常时，例如抛出 IgnoreRequest 异常，process_exception() 方法就会被调用。方法的返回值必须为 None、Response 对象、Request 对象之一。 process_exception() 方法的参数有如下三个。</p>
                  <ul>
                    <li>request，即 Request 对象，即产生异常的 Request</li>
                    <li>exception，即 Exception 对象，即抛出的异常</li>
                    <li>spdier，即 Spider 对象，即 Request 对应的 Spider</li>
                  </ul>
                  <p>下面归纳一下不同的返回值。</p>
                  <ul>
                    <li>当返回为 None 时，更低优先级的 Downloader Middleware 的 process_exception() 会被继续顺次调用，直到所有的方法都被调度完毕。</li>
                    <li>当返回为 Response 对象时，更低优先级的 Downloader Middleware 的 process_exception() 方法不再被继续调用，每个 Downloader Middleware 的 process_response() 方法转而被依次调用。</li>
                    <li>当返回为 Request 对象时，更低优先级的 Downloader Middleware 的 process_exception() 也不再被继续调用，该 Request 对象会重新放到调度队列里面等待被调度，它相当于一个全新的 Request。然后，该 Request 又会被 process_request() 方法顺次处理。</li>
                  </ul>
                  <p>以上内容便是这三个方法的详细使用逻辑。在使用它们之前，请先对这三个方法的返回值的处理情况有一个清晰的认识。在自定义 Downloader Middleware 的时候，也一定要注意每个方法的返回类型。 下面我们用一个案例实战来加深一下对 Downloader Middleware 用法的理解。</p>
                  <h3 id="3-项目实战"><a href="#3-项目实战" class="headerlink" title="3. 项目实战"></a>3. 项目实战</h3>
                  <p>新建一个项目，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject scrapydownloadertest</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建了一个 Scrapy 项目，名为 scrapydownloadertest。进入项目，新建一个 Spider，命令如下所示：</p>
                  <figure class="highlight avrasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy genspider httpbin httpbin<span class="meta">.org</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建了一个 Spider，名为 httpbin，源代码如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import scrapy</span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">HttpbinSpider</span>(<span class="title">scrapy</span>.<span class="title">Spider</span>):</span></span><br><span class="line">    name = <span class="string">'httpbin'</span></span><br><span class="line">    allowed_domains = [<span class="string">'httpbin.org'</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://httpbin.org/'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        pass</span><br><span class="line"><span class="string">``</span><span class="string">`接下来我们修改 start_urls 为：`</span>[<span class="string">'http://httpbin.org/'</span>]<span class="string">`。随后将 parse() 方法添加一行日志输出，将 response 变量的 text 属性输出出来，这样我们便可以看到 Scrapy 发送的 Request 信息了。</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">修改 Spider 内容如下所示：</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">`</span><span class="string">``</span>python</span><br><span class="line">import scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">HttpbinSpider</span>(<span class="title">scrapy</span>.<span class="title">Spider</span>):</span></span><br><span class="line">    name = <span class="string">'httpbin'</span></span><br><span class="line">    allowed_domains = [<span class="string">'httpbin.org'</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://httpbin.org/get'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.logger.debug(response.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来运行此 Spider，执行如下命令：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl httpbin</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>Scrapy 运行结果包含 Scrapy 发送的 Request 信息，内容如下所示：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip,deflate,br"</span>, </span><br><span class="line">    <span class="attr">"Accept-Language"</span>: <span class="string">"en"</span>, </span><br><span class="line">    <span class="attr">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"Scrapy/1.4.0 (+http://scrapy.org)"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们观察一下 Headers，Scrapy 发送的 Request 使用的 User-Agent 是 Scrapy/1.4.0(+<a href="http://scrapy.org" target="_blank" rel="noopener">http://scrapy.org)，这其实是由</a>，这其实是由) Scrapy 内置的 UserAgentMiddleware 设置的，UserAgentMiddleware 的源码如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy import signals</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">UserAgentMiddleware</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, user_agent=<span class="string">'Scrapy'</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.user_agent = user_agent</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        o = cls(crawler.settings[<span class="string">'USER_AGENT'</span>])</span><br><span class="line">        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)</span><br><span class="line">        <span class="keyword">return</span> o</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider_opened</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.user_agent = getattr(spider, <span class="string">'user_agent'</span>, <span class="keyword">self</span>.user_agent)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span><span class="params">(<span class="keyword">self</span>, request, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>.<span class="symbol">user_agent:</span></span><br><span class="line">            request.headers.setdefault(b<span class="string">'User-Agent'</span>, <span class="keyword">self</span>.user_agent)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 from_crawler() 方法中，首先尝试获取 settings 里面 USER_AGENT，然后把 USER_AGENT 传递给<strong>init</strong>() 方法进行初始化，其参数就是 user_agent。如果没有传递 USER_AGENT 参数就默认设置为 Scrapy 字符串。我们新建的项目没有设置 USER_AGENT，所以这里的 user_agent 变量就是 Scrapy。接下来，在 process_request() 方法中，将 user-agent 变量设置为 headers 变量的一个属性，这样就成功设置了 User-Agent。因此，User-Agent 就是通过此 Downloader Middleware 的 process_request() 方法设置的。 修改请求时的 User-Agent 可以有两种方式：一是修改 settings 里面的 USER_AGENT 变量；二是通过 Downloader Middleware 的 process_request() 方法来修改。 第一种方法非常简单，我们只需要在 setting.py 里面加一行 USER_AGENT 的定义即可：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">USER_AGENT = 'Mozilla/<span class="number">5.0</span> (Macintosh; Intel Mac OS X <span class="number">10</span>_12_6) AppleWebKit/<span class="number">537.36</span> (KHTML, like Gecko) Chrome/<span class="number">59.0</span><span class="number">.3071</span><span class="number">.115</span> Safari/<span class="number">537.36</span>'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>一般推荐使用此方法来设置。但是如果想设置得更灵活，比如设置随机的 User-Agent，那就需要借助 Downloader Middleware 了。所以接下来我们用 Downloader Middleware 实现一个随机 User-Agent 的设置。 在 middlewares.py 里面添加一个 RandomUserAgentMiddleware 的类，如下所示：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">RandomUserAgentMiddleware</span>():</span></span><br><span class="line"><span class="class">    def __init__(<span class="title">self</span>):</span></span><br><span class="line"><span class="class">        self.user_agents = ['<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span>; <span class="type">U</span>; <span class="type">MSIE</span> 9.0; <span class="type">Windows</span> <span class="type">NT</span> 9.0; <span class="title">en</span>-<span class="type">US</span>)',</span></span><br><span class="line"><span class="class">            '<span class="type">Mozilla</span>/5.0 (<span class="type">Windows</span> <span class="type">NT</span> 6.1) <span class="type">AppleWebKit</span>/537.2 (<span class="type">KHTML</span>, <span class="title">like</span> <span class="type">Gecko</span>) <span class="type">Chrome</span>/22.0.1216.0 <span class="type">Safari</span>/537.2',</span></span><br><span class="line"><span class="class">            '<span class="type">Mozilla</span>/5.0 (<span class="type">X11</span>; <span class="type">Ubuntu</span>; <span class="type">Linux</span> <span class="title">i686</span>; <span class="title">rv</span>:15.0) <span class="type">Gecko</span>/20100101 <span class="type">Firefox</span>/15.0.1'</span></span><br><span class="line"><span class="class">        ]</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">    def process_request(<span class="title">self</span>, <span class="title">request</span>, <span class="title">spider</span>):</span></span><br><span class="line"><span class="class">        request.headers['<span class="type">User</span>-<span class="type">Agent'</span>] = random.choice(<span class="title">self</span>.<span class="title">user_agents</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们首先在类的 <strong>init</strong>() 方法中定义了三个不同的 User-Agent，并用一个列表来表示。接下来实现了 process_request() 方法，它有一个参数 request，我们直接修改 request 的属性即可。在这里我们直接设置了 request 对象的 headers 属性的 User-Agent，设置内容是随机选择的 User-Agent，这样一个 Downloader Middleware 就写好了。 不过，要使之生效我们还需要再去调用这个 Downloader Middleware。在 settings.py 中，将 DOWNLOADER_MIDDLEWARES 取消注释，并设置成如下内容：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">DOWNLOADER_MIDDLEWARES</span> = &#123;<span class="string">'scrapydownloadertest.middlewares.RandomUserAgentMiddleware'</span>: <span class="number">543</span>,&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们重新运行 Spider，就可以看到 User-Agent 被成功修改为列表中所定义的随机的一个 User-Agent 了：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip,deflate,br"</span>, </span><br><span class="line">    <span class="attr">"Accept-Language"</span>: <span class="string">"en"</span>, </span><br><span class="line">    <span class="attr">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们就通过实现 Downloader Middleware 并利用 process_request() 方法成功设置了随机的 User-Agent。 另外，Downloader Middleware 还有 process_response() 方法。Downloader 对 Request 执行下载之后会得到 Response，随后 Scrapy 引擎会将 Response 发送回 Spider 进行处理。但是在 Response 被发送给 Spider 之前，我们同样可以使用 process_response() 方法对 Response 进行处理。比如这里修改一下 Response 的状态码，在 RandomUserAgentMiddleware 添加如下代码：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">process_response</span><span class="params">(<span class="keyword">self</span>, request, response, spider)</span></span><span class="symbol">:</span></span><br><span class="line">    response.status = <span class="number">201</span></span><br><span class="line">    <span class="keyword">return</span> response</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们将 response 对象的 status 属性修改为 201，随后将 response 返回，这个被修改后的 Response 就会被发送到 Spider。 我们再在 Spider 里面输出修改后的状态码，在 parse() 方法中添加如下的输出语句：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">self.logger.<span class="keyword">debug</span>('Status <span class="built_in">Code</span>: ' + <span class="built_in">str</span>(response.status))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>重新运行之后，控制台输出了如下内容：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[httpbin] <span class="keyword">DEBUG</span>: Status <span class="built_in">Code</span>: <span class="number">201</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，Response 的状态码成功修改了。 因此要想对 Response 进行后处理，就可以借助于 process_response() 方法。 另外还有一个 process_exception() 方法，它是用来处理异常的方法。如果需要异常处理的话，我们可以调用此方法。不过这个方法的使用频率相对低一些，在此不用实例演示。</p>
                  <h3 id="4-本节代码"><a href="#4-本节代码" class="headerlink" title="4. 本节代码"></a>4. 本节代码</h3>
                  <p>本节源代码为：<a href="https://github.com/Python3WebSpider/ScrapyDownloaderTest" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapyDownloaderTest</a>。</p>
                  <h3 id="5-结语"><a href="#5-结语" class="headerlink" title="5. 结语"></a>5. 结语</h3>
                  <p>本节讲解了 Downloader Middleware 的基本用法。此组件非常重要，是做异常处理和应对反爬处理的核心。后面我们会在实战中应用此组件来处理代理、Cookies 等内容。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-04 15:11:33" itemprop="dateCreated datePublished" datetime="2019-12-04T15:11:33+08:00">2019-12-04</time>
                </span>
                <span id="/8381.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.5–Downloader Middleware 的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8364.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8364.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.1–Scrapy 框架介绍</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-1-Scrapy-框架介绍"><a href="#13-1-Scrapy-框架介绍" class="headerlink" title="13.1 Scrapy 框架介绍"></a>13.1 Scrapy 框架介绍</h1>
                  <p>Scrapy 是一个基于 Twisted 的异步处理框架，是纯 Python 实现的爬虫框架，其架构清晰，模块之间的耦合程度低，可扩展性极强，可以灵活完成各种需求。我们只需要定制开发几个模块就可以轻松实现一个爬虫。</p>
                  <h3 id="1-架构介绍"><a href="#1-架构介绍" class="headerlink" title="1. 架构介绍"></a>1. 架构介绍</h3>
                  <p>首先我们来看下 Scrapy 框架的架构，如图 13-1 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-033839.jpg" alt=""> 图 13-1 Scrapy 架构 它可以分为如下的几个部分。</p>
                  <ul>
                    <li>Engine，引擎，用来处理整个系统的数据流处理，触发事务，是整个框架的核心。</li>
                    <li>Item，项目，它定义了爬取结果的数据结构，爬取的数据会被赋值成该对象。</li>
                    <li>Scheduler， 调度器，用来接受引擎发过来的请求并加入队列中，并在引擎再次请求的时候提供给引擎。</li>
                    <li>Downloader，下载器，用于下载网页内容，并将网页内容返回给蜘蛛。</li>
                    <li>Spiders，蜘蛛，其内定义了爬取的逻辑和网页的解析规则，它主要负责解析响应并生成提取结果和新的请求。</li>
                    <li>Item Pipeline，项目管道，负责处理由蜘蛛从网页中抽取的项目，它的主要任务是清洗、验证和存储数据。</li>
                    <li>Downloader Middlewares，下载器中间件，位于引擎和下载器之间的钩子框架，主要是处理引擎与下载器之间的请求及响应。</li>
                    <li>Spider Middlewares， 蜘蛛中间件，位于引擎和蜘蛛之间的钩子框架，主要工作是处理蜘蛛输入的响应和输出的结果及新的请求。</li>
                  </ul>
                  <h3 id="2-数据流"><a href="#2-数据流" class="headerlink" title="2. 数据流"></a>2. 数据流</h3>
                  <p>Scrapy 中的数据流由引擎控制，其过程如下:</p>
                  <ul>
                    <li>Engine 首先打开一个网站，找到处理该网站的 Spider 并向该 Spider 请求第一个要爬取的 URL。</li>
                    <li>Engine 从 Spider 中获取到第一个要爬取的 URL 并通过 Scheduler 以 Request 的形式调度。</li>
                    <li>Engine 向 Scheduler 请求下一个要爬取的 URL。</li>
                    <li>Scheduler 返回下一个要爬取的 URL 给 Engine，Engine 将 URL 通过 Downloader Middlewares 转发给 Downloader 下载。</li>
                    <li>一旦页面下载完毕， Downloader 生成一个该页面的 Response，并将其通过 Downloader Middlewares 发送给 Engine。</li>
                    <li>Engine 从下载器中接收到 Response 并通过 Spider Middlewares 发送给 Spider 处理。</li>
                    <li>Spider 处理 Response 并返回爬取到的 Item 及新的 Request 给 Engine。</li>
                    <li>Engine 将 Spider 返回的 Item 给 Item Pipeline，将新的 Request 给 Scheduler。</li>
                    <li>重复第二步到最后一步，直到 Scheduler 中没有更多的 Request，Engine 关闭该网站，爬取结束。</li>
                  </ul>
                  <p>通过多个组件的相互协作、不同组件完成工作的不同、组件对异步处理的支持，Scrapy 最大限度地利用了网络带宽，大大提高了数据爬取和处理的效率。</p>
                  <h3 id="3-项目结构"><a href="#3-项目结构" class="headerlink" title="3. 项目结构"></a>3. 项目结构</h3>
                  <p>Scrapy 框架和 pyspider 不同，它是通过命令行来创建项目的，代码的编写还是需要 IDE。项目创建之后，项目文件结构如下所示：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy.cfg</span><br><span class="line">project/</span><br><span class="line">    __init__.py</span><br><span class="line">    items.py</span><br><span class="line">    pipelines.py</span><br><span class="line">    settings.py</span><br><span class="line">    middlewares.py</span><br><span class="line">    spiders/</span><br><span class="line">        __init__.py</span><br><span class="line">        spider1.py</span><br><span class="line">        spider2.py</span><br><span class="line">        ...</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在此要将各个文件的功能描述如下：</p>
                  <ul>
                    <li>scrapy.cfg：它是 Scrapy 项目的配置文件，其内定义了项目的配置文件路径、部署相关信息等内容。</li>
                    <li>items.py：它定义 Item 数据结构，所有的 Item 的定义都可以放这里。</li>
                    <li>pipelines.py：它定义 Item Pipeline 的实现，所有的 Item Pipeline 的实现都可以放这里。</li>
                    <li>settings.py：它定义项目的全局配置。</li>
                    <li>middlewares.py：它定义 Spider Middlewares 和 Downloader Middlewares 的实现。</li>
                    <li>spiders：其内包含一个个 Spider 的实现，每个 Spider 都有一个文件。</li>
                  </ul>
                  <h3 id="4-结语"><a href="#4-结语" class="headerlink" title="4. 结语"></a>4. 结语</h3>
                  <p>本节介绍了 Scrapy 框架的基本架构、数据流过程以及项目结构。后面我们会详细了解 Scrapy 的用法，感受它的强大。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-03 11:42:17" itemprop="dateCreated datePublished" datetime="2019-12-03T11:42:17+08:00">2019-12-03</time>
                </span>
                <span id="/8364.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.1–Scrapy 框架介绍" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8361.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8361.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 9.4–ADSL 拨号代理</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="9-4-ADSL-拨号代理"><a href="#9-4-ADSL-拨号代理" class="headerlink" title="9.4 ADSL 拨号代理"></a>9.4 ADSL 拨号代理</h1>
                  <p>我们尝试维护过一个代理池。代理池可以挑选出许多可用代理，但是常常其稳定性不高、响应速度慢，而且这些代理通常是公共代理，可能不止一人同时使用，其 IP 被封的概率很大。另外，这些代理可能有效时间比较短，虽然代理池一直在筛选，但如果没有及时更新状态，也有可能获取到不可用的代理。 如果要追求更加稳定的代理，就需要购买专有代理或者自己搭建代理服务器。但是服务器一般都是固定的 IP，我们总不能搭建 100 个代理就用 100 台服务器吧，这显然是不现实的。 所以，ADSL 动态拨号主机就派上用场了。下面我们来了解一下 ADSL 拨号代理服务器的相关设置。</p>
                  <h3 id="1-什么是-ADSL"><a href="#1-什么是-ADSL" class="headerlink" title="1. 什么是 ADSL"></a>1. 什么是 ADSL</h3>
                  <p>ADSL（Asymmetric Digital Subscriber Line，非对称数字用户环路），它的上行和下行带宽不对称，它采用频分复用技术把普通的电话线分成了电话、上行和下行 3 个相对独立的信道，从而避免了相互之间的干扰。 ADSL 通过拨号的方式上网，需要输入 ADSL 账号和密码，每次拨号就更换一个 IP。IP 分布在多个 A 段，如果 IP 都能使用，则意味着 IP 量级可达千万。如果我们将 ADSL 主机作为代理，每隔一段时间主机拨号就换一个 IP，这样可以有效防止 IP 被封禁。另外，主机的稳定性很好，代理响应速度很快。</p>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>首先需要成功安装 Redis 数据库并启动服务，另外还需要安装 requests、redis-py、Tornado 库。如果没有安装，读者可以参考第一章的安装说明。</p>
                  <h3 id="3-购买主机"><a href="#3-购买主机" class="headerlink" title="3. 购买主机"></a>3. 购买主机</h3>
                  <p>我们先购买一台动态拨号 VPS 主机，这样的主机服务商相当多。在这里使用了云立方，官方网站：<a href="http://www.yunlifang.cn/dynamicvps.asp" target="_blank" rel="noopener">http://www.yunlifang.cn/dynamicvps.asp</a>。 建议选择电信线路。可以自行选择主机配置，主要考虑带宽是否满足需求。 然后进入拨号主机的后台，预装一个操作系统，如图 9-10 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052822.jpg" alt=""> 图 9-10 预装操作系统 推荐安装 CentOS 7 系统。 然后找到远程管理面板远程连接的用户名和密码，也就是 SSH 远程连接服务器的信息。比如我使用的 IP 和端口是 153.36.65.214:20063，用户名是 root。命令行下输入如下内容：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh <span class="symbol">root@</span><span class="number">153.36</span><span class="number">.65</span><span class="number">.214</span> -p <span class="number">20063</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输入管理密码，就可以连接上远程服务器了。 进入之后，我们发现一个可用的脚本文件 ppp.sh，这是拨号初始化的脚本。运行此脚本会提示输入拨号的用户名和密码，然后它就开始各种拨号配置。一次配置成功，后面拨号就不需要重复输入用户名和密码。 运行 ppp.sh 脚本，输入用户名、密码等待它的配置完成，如图 9-11 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052827.jpg" alt=""> 图 9-11 配置页面 提示成功之后就可以进行拨号了。注意，在拨号之前测试 ping 任何网站都是不通的，因为当前网络还没联通。输入如下拨号命令：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">adsl-<span class="literal">start</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>拨号命令成功运行，没有报错信息，耗时约几秒。接下来再去 ping 外网就可以通了。 如果要停止拨号，可以输入如下命令：</p>
                  <figure class="highlight arduino">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">adsl-<span class="built_in">stop</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之后，可以发现又连不通网络了，如图 9-12 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052841.jpg" alt=""> 图 9-12 拨号建立连接 断线重播的命令就是二者组合起来，先执行 adsl-stop，再执行 adsl-start。每次拨号，ifconfig 命令观察主机的 IP，发现主机的 IP 一直在变化，网卡名称叫作 ppp0，如图 9-13 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052851.jpg" alt=""> 图 9-13 网络设备信息 接下来，我们要做两件事：一是怎样将主机设置为代理服务器，二是怎样实时获取拨号主机的 IP。</p>
                  <h3 id="4-设置代理服务器"><a href="#4-设置代理服务器" class="headerlink" title="4. 设置代理服务器"></a>4. 设置代理服务器</h3>
                  <p>在 Linux 下搭建 HTTP 代理服务器，推荐 TinyProxy 和 Squid，配置都非常简单。在这里我们以 TinyProxy 为例来讲解一下怎样搭建代理服务器。</p>
                  <h4 id="安装-TinyProxy"><a href="#安装-TinyProxy" class="headerlink" title="安装 TinyProxy"></a>安装 TinyProxy</h4>
                  <p>第一步就是安装 TinyProxy 软件。在这里我使用的系统是 CentOS，所以使用 yum 来安装。如果是其他系统，如 Ubuntu，可以选择 apt-get 等命令安装。 命令行执行 yum 安装指令：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yum <span class="keyword">install</span> -y epel-<span class="keyword">release</span></span><br><span class="line">yum <span class="keyword">update</span> -y</span><br><span class="line">yum <span class="keyword">install</span> -y tinyproxy</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行完成之后就可以完成 tinyproxy 的安装了。</p>
                  <h4 id="配置-TinyProxy"><a href="#配置-TinyProxy" class="headerlink" title="配置 TinyProxy"></a>配置 TinyProxy</h4>
                  <p>TinyProxy 安装完成之后还要配置一下才可以用作代理服务器。我们需要编辑配置文件，此文件一般的路径是 /etc/tinyproxy/tinyproxy.conf。 可以看到有一行</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Port <span class="number">8888</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里可以设置代理的端口，默认是 8888。 继续向下找到如下代码：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Allow <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这行代码表示被允许连接的主机 IP。如果希望连接任何主机，那就直接将这行代码注释即可。在这里我们选择直接注释，也就是任何主机都可以使用这台主机作为代理服务器。 修改为如下代码：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Allow <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>设置完成之后重启 TinyProxy 即可：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">systemctl</span> <span class="selector-tag">enable</span> <span class="selector-tag">tinyproxy</span><span class="selector-class">.service</span></span><br><span class="line"><span class="selector-tag">systemctl</span> <span class="selector-tag">restart</span> <span class="selector-tag">tinyproxy</span><span class="selector-class">.service</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>防火墙开放该端口：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">iptables -I <span class="keyword">INPUT</span> -p tcp <span class="comment">--dport 8888 -j ACCEPT</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然如果想直接关闭防火墙也可以：</p>
                  <figure class="highlight arduino">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">systemctl <span class="built_in">stop</span> firewalld.service</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就完成了 TinyProxy 的配置了。</p>
                  <h4 id="验证-TinyProxy"><a href="#验证-TinyProxy" class="headerlink" title="验证 TinyProxy"></a>验证 TinyProxy</h4>
                  <p>首先，用 ifconfig 查看当前主机的 IP。比如，当前我的主机拨号 IP 为 112.84.118.216，在其他的主机运行测试一下。 用 curl 命令设置代理请求 httpbin，检测代理是否生效。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl -x <span class="number">112.84</span><span class="number">.118</span><span class="number">.216</span>:<span class="number">8888</span> httpbin.org/<span class="keyword">get</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图 9-14 所示： <img src="https://qiniu.cuiqingcai.com/2019-10-20-052902.jpg" alt=""> 图 9-14 运行结果 如果有正常的结果输出，并且 origin 的值为代理 IP 的地址，就证明 TinyProxy 配置成功了。</p>
                  <h3 id="5-动态获取-IP"><a href="#5-动态获取-IP" class="headerlink" title="5. 动态获取 IP"></a>5. 动态获取 IP</h3>
                  <p>现在可以执行命令让主机动态切换 IP，也在主机上搭建了代理服务器。我们只需要知道拨号后的 IP 就可以使用代理。 我们考虑到，在一台主机拨号切换 IP 的间隙代理是不可用的，在这拨号的几秒时间内如果有第二台主机顶替第一台主机，那就可以解决拨号间隙代理无法使用的问题了。所以我们要设计的架构必须要考虑支持多主机的问题。 假如有 10 台拨号主机同时需要维护，而爬虫需要使用这 10 台主机的代理，那么在爬虫端维护的开销是非常大的。如果爬虫在不同的机器上运行，那么每个爬虫必须要获得这 10 台拨号主机的配置，这显然是不理想的。 为了更加方便地使用代理，我们可以像上文的代理池一样定义一个统一的代理接口，爬虫端只需要配置代理接口即可获取可用代理。要搭建一个接口，就势必需要一台服务器，而接口的数据从哪里获得呢，当然最理想的还是选择数据库。 比如我们需要同时维护 10 台拨号主机，每台拨号主机都会定时拨号，那这样每台主机在某个时刻可用的代理只有一个，所以我们没有必要存储之前的拨号代理，因为重新拨号之后之前的代理已经不能用了，所以只需要将之前的代理更新其内容就好了。数据库要做的就是定时对每台主机的代理进行更新，而更新时又需要拨号主机的唯一标识，根据主机标识查出这条数据，然后将这条数据对应的代理更新。 所以数据库端就需要存储一个主机标识到代理的映射关系。那么很自然地我们就会想到关系型数据库，如 MySQL 或者 Redis 的 Hash 存储，只需存储一个映射关系，不需要很多字段，而且 Redis 比 MySQL 效率更高、使用更方便，所以最终选定的存储方式就是 Redis 的 Hash。</p>
                  <h3 id="6-存储模块"><a href="#6-存储模块" class="headerlink" title="6. 存储模块"></a>6. 存储模块</h3>
                  <p>那么接下来我们要做可被远程访问的 Redis 数据库，各个拨号机器只需要将各自的主机标识和当前 IP 和端口（也就是代理）发送给数据库就好了。 先定义一个操作 Redis 数据库的类，示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> redis</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="comment"># Redis 数据库 IP</span></span><br><span class="line">REDIS_HOST = <span class="string">'remoteaddress'</span></span><br><span class="line"><span class="comment"># Redis 数据库密码，如无则填 None</span></span><br><span class="line">REDIS_PASSWORD = <span class="string">'foobared'</span></span><br><span class="line"><span class="comment"># Redis 数据库端口</span></span><br><span class="line">REDIS_PORT = <span class="number">6379</span></span><br><span class="line"><span class="comment"># 代理池键名</span></span><br><span class="line">PROXY_KEY = <span class="string">'adsl'</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RedisClient</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, proxy_key=PROXY_KEY)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        初始化 Redis 连接</span></span><br><span class="line"><span class="string">        :param host: Redis 地址</span></span><br><span class="line"><span class="string">        :param port: Redis 端口</span></span><br><span class="line"><span class="string">        :param password: Redis 密码</span></span><br><span class="line"><span class="string">        :param proxy_key: Redis 哈希表名</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=<span class="literal">True</span>)</span><br><span class="line">        self.proxy_key = proxy_key</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">set</span><span class="params">(self, name, proxy)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        设置代理</span></span><br><span class="line"><span class="string">        :param name: 主机名称</span></span><br><span class="line"><span class="string">        :param proxy: 代理</span></span><br><span class="line"><span class="string">        :return: 设置结果</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hset(self.proxy_key, name, proxy)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get</span><span class="params">(self, name)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        获取代理</span></span><br><span class="line"><span class="string">        :param name: 主机名称</span></span><br><span class="line"><span class="string">        :return: 代理</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hget(self.proxy_key, name)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">count</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        获取代理总数</span></span><br><span class="line"><span class="string">        :return: 代理总数</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hlen(self.proxy_key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">remove</span><span class="params">(self, name)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        删除代理</span></span><br><span class="line"><span class="string">        :param name: 主机名称</span></span><br><span class="line"><span class="string">        :return: 删除结果</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hdel(self.proxy_key, name)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">names</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        获取主机名称列表</span></span><br><span class="line"><span class="string">        :return: 获取主机名称列表</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hkeys(self.proxy_key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">proxies</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        获取代理列表</span></span><br><span class="line"><span class="string">        :return: 代理列表</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">return</span> self.db.hvals(self.proxy_key)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">random</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        随机获取代理</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        proxies = self.proxies()</span><br><span class="line">        <span class="keyword">return</span> random.choice(proxies)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">all</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        获取字典</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        """</span><span class="keyword">return</span> self.db.hgetall(self.proxy_key)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了一个 RedisClient 类，在<strong>init</strong>() 方法中初始化了 Redis 连接，其中 REDIS_HOST 就是远程 Redis 的地址，REDIS_PASSWORD 是密码，REDIS_PORT 是端口，PROXY_KEY 是存储代理的散列表的键名。 接下来定义了一个 set() 方法，这个方法用来向散列表添加映射关系。映射是从主机标识到代理的映射，比如一台主机的标识为 adsl1，当前的代理为 118.119.111.172:8888，那么散列表中就会存储一个 key 为 adsl1、value 为 118.119.111.172:8888 的映射，Hash 结构如图 9-15 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052948.jpg" alt=""> 图 9-15 Hash 结构 如果有多台主机，只需要向 Hash 中添加映射即可。 另外，get() 方法就是从散列表中取出某台主机对应的代理。remove() 方法则是从散列表中移除对应的主机的代理。还有 names()、proxies()、all() 方法则是分别获取散列表中的主机列表、代理列表及所有主机代理映射。count() 方法则是返回当前散列表的大小，也就是可用代理的数目。 最后还有一个比较重要的方法 random()，它随机从散列表中取出一个可用代理，类似前面代理池的思想，确保每个代理都能被取到。 如果要对数据库进行操作，只需要初始化 RedisClient 对象，然后调用它的 set() 或者 remove() 方法，即可对散列表进行设置和删除。</p>
                  <h3 id="7-拨号模块"><a href="#7-拨号模块" class="headerlink" title="7. 拨号模块"></a>7. 拨号模块</h3>
                  <p>接下来要做的就是拨号，并把新的 IP 保存到 Redis 散列表里。 首先是拨号定时，它分为定时拨号和非定时拨号两种选择。 非定时拨号：最好的方法就是向该主机发送一个信号，然后主机就启动拨号，但这样做的话，我们首先要搭建一个重新拨号的接口，如搭建一个 Web 接口，请求该接口即进行拨号，但开始拨号之后，此时主机的状态就从在线转为离线，而此时的 Web 接口也就相应失效了，拨号过程无法再连接，拨号之后接口的 IP 也变了，所以我们无法通过接口来方便地控制拨号过程和获取拨号结果，下次拨号还得改变拨号请求接口，所以非定时拨号的开销还是比较大的。 定时拨号：我们只需要在拨号主机上运行定时脚本即可，每隔一段时间拨号一次，更新 IP，然后将 IP 在 Redis 散列表中更新即可，非常简单易用，另外可以适当将拨号频率调高一点，减少短时间内 IP 被封的可能性。 在这里选择定时拨号。 接下来就是获取 IP。获取拨号后的 IP 非常简单，只需要调用 ifconfig 命令，然后解析出对应网卡的 IP 即可。 获取了 IP 之后，我们还需要进行有效性检测。拨号主机可以自己检测，比如可以利用 requests 设置自身的代理请求外网，如果成功，那么证明代理可用，然后再修改 Redis 散列表，更新代理。 需要注意，由于在拨号的间隙拨号主机是离线状态，而此时 Redis 散列表中还存留了上次的代理，一旦这个代理被取用了，该代理是无法使用的。为了避免这个情况，每台主机在拨号之前还需要将自身的代理从 Redis 散列表中移除。 这样基本的流程就理顺了，我们用如下代码实现：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line">import time</span><br><span class="line">import requests</span><br><span class="line"><span class="keyword">from</span> requests.exceptions import ConnectionError, ReadTimeout</span><br><span class="line"><span class="keyword">from</span> db import RedisClient</span><br><span class="line"></span><br><span class="line"><span class="comment"># 拨号网卡</span></span><br><span class="line">ADSL_IFNAME = <span class="string">'ppp0'</span></span><br><span class="line"><span class="comment"># 测试 URL</span></span><br><span class="line">TEST_URL = <span class="string">'http://www.baidu.com'</span></span><br><span class="line"><span class="comment"># 测试超时时间</span></span><br><span class="line">TEST_TIMEOUT = 20</span><br><span class="line"><span class="comment"># 拨号间隔</span></span><br><span class="line">ADSL_CYCLE = 100</span><br><span class="line"><span class="comment"># 拨号出错重试间隔</span></span><br><span class="line">ADSL_ERROR_CYCLE = 5</span><br><span class="line"><span class="comment"># ADSL 命令</span></span><br><span class="line">ADSL_BASH = <span class="string">'adsl-stop;adsl-start'</span></span><br><span class="line"><span class="comment"># 代理运行端口</span></span><br><span class="line">PROXY_PORT = 8888</span><br><span class="line"><span class="comment"># 客户端唯一标识</span></span><br><span class="line">CLIENT_NAME = <span class="string">'adsl1'</span></span><br><span class="line"></span><br><span class="line">class Sender():</span><br><span class="line">    def get_ip(self, <span class="attribute">ifname</span>=ADSL_IFNAME):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        获取本机 IP</span></span><br><span class="line"><span class="string">        :param ifname: 网卡名称</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        (status, output) = subprocess.getstatusoutput(<span class="string">'ifconfig'</span>)</span><br><span class="line">        <span class="keyword">if</span> status == 0:</span><br><span class="line">            pattern = re.compile(ifname + <span class="string">'.*?inet.*?(d+.d+.d+.d+).*?netmask'</span>, re.S)</span><br><span class="line">            result = re.search(pattern, output)</span><br><span class="line">            <span class="keyword">if</span> result:</span><br><span class="line">               <span class="built_in"> ip </span>= result.group(1)</span><br><span class="line">                return ip</span><br><span class="line"></span><br><span class="line">    def test_proxy(self, proxy):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        测试代理</span></span><br><span class="line"><span class="string">        :param proxy: 代理</span></span><br><span class="line"><span class="string">        :return: 测试结果</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        try:</span><br><span class="line">            response = requests.<span class="builtin-name">get</span>(TEST_URL, proxies=&#123;</span><br><span class="line">                <span class="string">'http'</span>: <span class="string">'http://'</span> + proxy,</span><br><span class="line">                <span class="string">'https'</span>: <span class="string">'https://'</span> + proxy</span><br><span class="line">            &#125;, <span class="attribute">timeout</span>=TEST_TIMEOUT)</span><br><span class="line">            <span class="keyword">if</span> response.status_code == 200:</span><br><span class="line">                return <span class="literal">True</span></span><br><span class="line">        except (ConnectionError, ReadTimeout):</span><br><span class="line">            return <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">    def remove_proxy(self):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        移除代理</span></span><br><span class="line"><span class="string">        :return: None</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        self.redis = RedisClient()</span><br><span class="line">        self.redis.<span class="builtin-name">remove</span>(CLIENT_NAME)</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'Successfully Removed Proxy'</span>)</span><br><span class="line"></span><br><span class="line">    def set_proxy(self, proxy):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        设置代理</span></span><br><span class="line"><span class="string">        :param proxy: 代理</span></span><br><span class="line"><span class="string">        :return: None</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        self.redis = RedisClient()</span><br><span class="line">        <span class="keyword">if</span> self.redis.<span class="builtin-name">set</span>(CLIENT_NAME, proxy):</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Successfully Set Proxy'</span>, proxy)</span><br><span class="line"></span><br><span class="line">    def adsl(self):</span><br><span class="line">        <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">        拨号主进程</span></span><br><span class="line"><span class="string">        :return: None</span></span><br><span class="line"><span class="string">        "</span><span class="string">""</span></span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'ADSL Start, Remove Proxy, Please wait'</span>)</span><br><span class="line">            self.remove_proxy()</span><br><span class="line">            (status, output) = subprocess.getstatusoutput(ADSL_BASH)</span><br><span class="line">            <span class="keyword">if</span> status == 0:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'ADSL Successfully'</span>)</span><br><span class="line">               <span class="built_in"> ip </span>= self.get_ip()</span><br><span class="line">                <span class="keyword">if</span> ip:</span><br><span class="line">                    <span class="builtin-name">print</span>(<span class="string">'Now IP'</span>, ip)</span><br><span class="line">                    <span class="builtin-name">print</span>(<span class="string">'Testing Proxy, Please Wait'</span>)</span><br><span class="line">                   <span class="built_in"> proxy </span>= <span class="string">'&#123;ip&#125;:&#123;port&#125;'</span>.format(<span class="attribute">ip</span>=ip, <span class="attribute">port</span>=PROXY_PORT)</span><br><span class="line">                    <span class="keyword">if</span> self.test_proxy(proxy):</span><br><span class="line">                        <span class="builtin-name">print</span>(<span class="string">'Valid Proxy'</span>)</span><br><span class="line">                        self.set_proxy(proxy)</span><br><span class="line">                        <span class="builtin-name">print</span>(<span class="string">'Sleeping'</span>)</span><br><span class="line">                        time.sleep(ADSL_CYCLE)</span><br><span class="line">                    <span class="keyword">else</span>:</span><br><span class="line">                        <span class="builtin-name">print</span>(<span class="string">'Invalid Proxy'</span>)</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    <span class="builtin-name">print</span>(<span class="string">'Get IP Failed, Re Dialing'</span>)</span><br><span class="line">                    time.sleep(ADSL_ERROR_CYCLE)</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'ADSL Failed, Please Check'</span>)</span><br><span class="line">                time.sleep(ADSL_ERROR_CYCLE)</span><br><span class="line">def <span class="builtin-name">run</span>():</span><br><span class="line">    sender = Sender()</span><br><span class="line">    sender.adsl()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里定义了一个 Sender 类，它的主要作用是执行定时拨号，并将新的 IP 测试通过之后更新到远程 Redis 散列表里。 主方法是 adsl() 方法，它首先是一个无限循环，循环体内就是拨号的逻辑。 adsl() 方法首先调用了 remove_proxy() 方法，将远程 Redis 散列表中本机对应的代理移除，避免拨号时本主机的残留代理被取到。 接下来利用 subprocess 模块来执行拨号脚本，拨号脚本很简单，就是 stop 之后再 start，这里将拨号的命令直接定义成了 ADSL_BASH。 随后程序又调用 get_ip() 方法，通过 subprocess 模块执行获取 IP 的命令 ifconfig，然后根据网卡名称获取了当前拨号网卡的 IP 地址，即拨号后的 IP。 再接下来就需要测试代理有效性了。程序首先调用了 test_proxy() 方法，将自身的代理设置好，使用 requests 库来用代理连接 TEST_URL。在此 TEST_URL 设置为百度，如果请求成功，则证明代理有效。 如果代理有效，再调用 set_proxy() 方法将 Redis 散列表中本机对应的代理更新，设置时需要指定本机唯一标识和本机当前代理。本机唯一标识可随意配置，其对应的变量为 CLIENT_NAME，保证各台拨号主机不冲突即可。本机当前代理则由拨号后的新 IP 加端口组合而成。通过调用 RedisClient 的 set() 方法，参数 name 为本机唯一标识，proxy 为拨号后的新代理，执行之后便可以更新散列表中的本机代理了。 建议至少配置两台主机，这样在一台主机的拨号间隙还有另一台主机的代理可用。拨号主机的数量不限，越多越好。 在拨号主机上执行拨号脚本，示例输出如图 9-16 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-052912.jpg" alt=""> 图 9-16 示例输出 首先移除了代理，再进行拨号，拨号完成之后获取新的 IP，代理检测成功之后就设置到 Redis 散列表中，然后等待一段时间再重新进行拨号。 我们添加了多台拨号主机，这样就有多个稳定的定时更新的代理可用了。Redis 散列表会实时更新各台拨号主机的代理，如图 9-17 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-053042.jpg" alt=""> 图 9-17 Hash 结构 图中所示是四台 ADSL 拨号主机配置并运行后的散列表的内容，表中的代理都是可用的。</p>
                  <h3 id="8-接口模块"><a href="#8-接口模块" class="headerlink" title="8. 接口模块"></a>8. 接口模块</h3>
                  <p>目前为止，我们已经成功实时更新拨号主机的代理。不过还缺少一个模块，那就是接口模块。像之前的代理池一样，我们也定义一些接口来获取代理，如 random 获取随机代理、count 获取代理个数等。 我们选用 Tornado 来实现，利用 Tornado 的 Server 模块搭建 Web 接口服务，示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> tornado.ioloop</span><br><span class="line"><span class="keyword">import</span> tornado.web</span><br><span class="line"><span class="keyword">from</span> tornado.web <span class="keyword">import</span> RequestHandler, Application</span><br><span class="line"></span><br><span class="line"><span class="comment"># API 端口</span></span><br><span class="line">API_PORT = <span class="number">8000</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MainHandler</span><span class="params">(RequestHandler)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">initialize</span><span class="params">(self, redis)</span>:</span></span><br><span class="line">        self.redis = redis</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get</span><span class="params">(self, api=<span class="string">''</span>)</span>:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> api:</span><br><span class="line">            links = [<span class="string">'random'</span>, <span class="string">'proxies'</span>, <span class="string">'names'</span>, <span class="string">'all'</span>, <span class="string">'count'</span>]</span><br><span class="line">            self.write(<span class="string">'&lt;h4&gt;Welcome to ADSL Proxy API&lt;/h4&gt;'</span>)</span><br><span class="line">            <span class="keyword">for</span> link <span class="keyword">in</span> links:</span><br><span class="line">                self.write(<span class="string">'&lt;a href='</span> + link + <span class="string">'&gt;'</span> + link + <span class="string">'&lt;/a&gt;&lt;br&gt;'</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> api == <span class="string">'random'</span>:</span><br><span class="line">            result = self.redis.random()</span><br><span class="line">            <span class="keyword">if</span> result:</span><br><span class="line">                self.write(result)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> api == <span class="string">'names'</span>:</span><br><span class="line">            result = self.redis.names()</span><br><span class="line">            <span class="keyword">if</span> result:</span><br><span class="line">                self.write(json.dumps(result))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> api == <span class="string">'proxies'</span>:</span><br><span class="line">            result = self.redis.proxies()</span><br><span class="line">            <span class="keyword">if</span> result:</span><br><span class="line">                self.write(json.dumps(result))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> api == <span class="string">'all'</span>:</span><br><span class="line">            result = self.redis.all()</span><br><span class="line">            <span class="keyword">if</span> result:</span><br><span class="line">                self.write(json.dumps(result))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> api == <span class="string">'count'</span>:</span><br><span class="line">            self.write(str(self.redis.count()))</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">server</span><span class="params">(redis, port=API_PORT, address=<span class="string">''</span>)</span>:</span></span><br><span class="line">    application = Application([(<span class="string">r'/'</span>, MainHandler, dict(redis=redis)),</span><br><span class="line">        (<span class="string">r'/(.*)'</span>, MainHandler, dict(redis=redis)),</span><br><span class="line">    ])</span><br><span class="line">    application.listen(port, address=address)</span><br><span class="line">    print(<span class="string">'ADSL API Listening on'</span>, port)</span><br><span class="line">    tornado.ioloop.IOLoop.instance().start()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了 5 个接口，random 获取随机代理，names 获取主机列表，proxies 获取代理列表，all 获取代理映射，count 获取代理数量。 程序启动之后便会在 API_PORT 端口上运行 Web 服务，主页面如图 9-18 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-053047.jpg" alt=""> 图 9-18 主页面 访问 proxies 接口可以获得所有代理列表，如图 9-19 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-053052.jpg" alt=""> 图 9-19 代理列表 访问 random 接口可以获取随机可用代理，如图 9-20 所示。 <img src="https://qiniu.cuiqingcai.com/2019-10-20-053057.jpg" alt=""> 图 9-20 随机代理 我们只需将接口部署到服务器上，即可通过 Web 接口获取可用代理，获取方式和代理池类似。</p>
                  <h3 id="9-本节代码"><a href="#9-本节代码" class="headerlink" title="9. 本节代码"></a>9. 本节代码</h3>
                  <p>本节代码地址为：<a href="https://github.com/Python3WebSpider/AdslProxy" target="_blank" rel="noopener">https://github.com/Python3WebSpider/AdslProxy</a>。</p>
                  <h3 id="10-结语"><a href="#10-结语" class="headerlink" title="10. 结语"></a>10. 结语</h3>
                  <p>本节介绍了 ADSL 拨号代理的搭建过程。通过这种代理，我们可以无限次更换 IP，而且线路非常稳定，抓取效果好很多。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-03 11:07:04" itemprop="dateCreated datePublished" datetime="2019-12-03T11:07:04+08:00">2019-12-03</time>
                </span>
                <span id="/8361.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 9.4–ADSL 拨号代理" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>10k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8353.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8353.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.4–Spider 的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-4-Spider-的用法"><a href="#13-4-Spider-的用法" class="headerlink" title="13.4 Spider 的用法"></a>13.4 Spider 的用法</h1>
                  <p>在 Scrapy 中，要抓取网站的链接配置、抓取逻辑、解析逻辑里其实都是在 Spider 中配置的。在前一节实例中，我们发现抓取逻辑也是在 Spider 中完成的。本节我们就来专门了解一下 Spider 的基本用法。</p>
                  <h3 id="1-Spider-运行流程"><a href="#1-Spider-运行流程" class="headerlink" title="1. Spider 运行流程"></a>1. Spider 运行流程</h3>
                  <p>在实现 Scrapy 爬虫项目时，最核心的类便是 Spider 类了，它定义了如何爬取某个网站的流程和解析方式。简单来讲，Spider 要做的事就是如下两件。</p>
                  <ul>
                    <li>定义爬取网站的动作</li>
                    <li>分析爬取下来的网页</li>
                  </ul>
                  <p>对于 Spider 类来说，整个爬取循环如下所述。</p>
                  <ul>
                    <li>以初始的 URL 初始化 Request，并设置回调函数。 当该 Request 成功请求并返回时，将生成 Response，并作为参数传给该回调函数。</li>
                    <li>在回调函数内分析返回的网页内容。返回结果可以有两种形式，一种是解析到的有效结果返回字典或 Item 对象。下一步可经过处理后（或直接）保存，另一种是解析得下一个（如下一页）链接，可以利用此链接构造 Request 并设置新的回调函数，返回 Request。</li>
                    <li>如果返回的是字典或 Item 对象，可通过 Feed Exports 等形式存入到文件，如果设置了 Pipeline 的话，可以经由 Pipeline 处理（如过滤、修正等）并保存。</li>
                    <li>如果返回的是 Reqeust，那么 Request 执行成功得到 Response 之后会再次传递给 Request 中定义的回调函数，可以再次使用选择器来分析新得到的网页内容，并根据分析的数据生成 Item。</li>
                  </ul>
                  <p>通过以上几步循环往复进行，便完成了站点的爬取。</p>
                  <h3 id="2-Spider-类分析"><a href="#2-Spider-类分析" class="headerlink" title="2. Spider 类分析"></a>2. Spider 类分析</h3>
                  <p>在上一节的例子中我们定义的 Spider 是继承自 scrapy.spiders.Spider，这个类是最简单最基本的 Spider 类，每个其他的 Spider 必须继承这个类，还有后文要说明的一些特殊 Spider 类也都是继承自它。 这个类里提供了 start_requests() 方法的默认实现，读取并请求 start_urls 属性，并根据返回的结果调用 parse() 方法解析结果。另外它还有一些基础属性，下面对其进行讲解：</p>
                  <ul>
                    <li>name，爬虫名称，是定义 Spider 名字的字符串。Spider 的名字定义了 Scrapy 如何定位并初始化 Spider，所以其必须是唯一的。 不过我们可以生成多个相同的 Spider 实例，这没有任何限制。 name 是 Spider 最重要的属性，而且是必须的。如果该 Spider 爬取单个网站，一个常见的做法是以该网站的域名名称来命名 Spider。 例如，如果 Spider 爬取 mywebsite.com ，该 Spider 通常会被命名为 mywebsite 。</li>
                    <li>allowed_domains，允许爬取的域名，是可选配置，不在此范围的链接不会被跟进爬取。</li>
                    <li>start_urls，起始 URL 列表，当我们没有实现 start_requests() 方法时，默认会从这个列表开始抓取。</li>
                    <li>custom_settings，这是一个字典，是专属于本 Spider 的配置，此设置会覆盖项目全局的设置，而且此设置必须在初始化前被更新，所以它必须定义成类变量。</li>
                    <li>crawler，此属性是由 from_crawler() 方法设置的，代表的是本 Spider 类对应的 Crawler 对象，Crawler 对象中包含了很多项目组件，利用它我们可以获取项目的一些配置信息，如最常见的就是获取项目的设置信息，即 Settings。</li>
                    <li>settings，是一个 Settings 对象，利用它我们可以直接获取项目的全局设置变量。</li>
                  </ul>
                  <p>除了一些基础属性，Spider 还有一些常用的方法，在此介绍如下：</p>
                  <ul>
                    <li>start_requests()，此方法用于生成初始请求，它必须返回一个可迭代对象，此方法会默认使用 start_urls 里面的 URL 来构造 Request，而且 Request 是 GET 请求方式。如果我们想在启动时以 POST 方式访问某个站点，可以直接重写这个方法，发送 POST 请求时我们使用 FormRequest 即可。</li>
                    <li>parse()，当 Response 没有指定回调函数时，该方法会默认被调用，它负责处理 Response，处理返回结果，并从中提取出想要的数据和下一步的请求，然后返回。该方法需要返回一个包含 Request 或 Item 的可迭代对象。</li>
                    <li>closed()，当 Spider 关闭时，该方法会被调用，在这里一般会定义释放资源的一些操作或其他收尾操作。</li>
                  </ul>
                  <h3 id="3-结语"><a href="#3-结语" class="headerlink" title="3. 结语"></a>3. 结语</h3>
                  <p>以上的介绍可能初看起来有点摸不清头脑，不过不用担心，后面我们会有很多实例来使用这些属性和方法，慢慢会熟练掌握的。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-03 09:40:04" itemprop="dateCreated datePublished" datetime="2019-12-03T09:40:04+08:00">2019-12-03</time>
                </span>
                <span id="/8353.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.4–Spider 的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8350.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8350.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.3–Selector 的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-3-Selector-的用法"><a href="#13-3-Selector-的用法" class="headerlink" title="13.3 Selector 的用法"></a>13.3 Selector 的用法</h1>
                  <p>我们之前介绍了利用 Beautiful Soup、pyquery 以及正则表达式来提取网页数据，这确实非常方便。而 Scrapy 还提供了自己的数据提取方法，即 Selector（选择器）。Selector 是基于 lxml 来构建的，支持 XPath 选择器、CSS 选择器以及正则表达式，功能全面，解析速度和准确度非常高。 本节将介绍 Selector 的用法。</p>
                  <h3 id="1-直接使用"><a href="#1-直接使用" class="headerlink" title="1. 直接使用"></a>1. 直接使用</h3>
                  <p>Selector 是一个可以独立使用的模块。我们可以直接利用 Selector 这个类来构建一个选择器对象，然后调用它的相关方法如 xpath()、css() 等来提取数据。 例如，针对一段 HTML 代码，我们可以用如下方式构建 Selector 对象来提取数据：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> scrapy import Selector</span><br><span class="line"></span><br><span class="line">body = <span class="comment">'<span class="doctag">&lt;html&gt;</span><span class="doctag">&lt;head&gt;</span><span class="doctag">&lt;title&gt;</span>Hello World<span class="doctag">&lt;/title&gt;</span><span class="doctag">&lt;/head&gt;</span><span class="doctag">&lt;body&gt;</span><span class="doctag">&lt;/body&gt;</span><span class="doctag">&lt;/html&gt;</span>'</span></span><br><span class="line">selector = Selector(<span class="keyword">text</span>=body)</span><br><span class="line">title = selector.xpath(<span class="comment">'//title/text()').extract_first()</span></span><br><span class="line">print(title)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Hello World</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们在这里没有在 Scrapy 框架中运行，而是把 Scrapy 中的 Selector 单独拿出来使用了，构建的时候传入 text 参数，就生成了一个 Selector 选择器对象，然后就可以像前面我们所用的 Scrapy 中的解析方式一样，调用 xpath()、css() 等方法来提取了。 在这里我们查找的是源代码中的 title 中的文本，在 XPath 选择器最后加 text() 方法就可以实现文本的提取了。 以上内容就是 Selector 的直接使用方式。同 Beautiful Soup 等库类似，Selector 其实也是强大的网页解析库。如果方便的话，我们也可以在其他项目中直接使用 Selector 来提取数据。 接下来，我们用实例来详细讲解 Selector 的用法。</p>
                  <h3 id="2-Scrapy-Shell"><a href="#2-Scrapy-Shell" class="headerlink" title="2. Scrapy Shell"></a>2. Scrapy Shell</h3>
                  <p>由于 Selector 主要是与 Scrapy 结合使用，如 Scrapy 的回调函数中的参数 response 直接调用 xpath() 或者 css() 方法来提取数据，所以在这里我们借助 Scrapy shell 来模拟 Scrapy 请求的过程，来讲解相关的提取方法。 我们用官方文档的一个样例页面来做演示：<a href="http://doc.scrapy.org/en/latest/_static/selectors-sample1.html" target="_blank" rel="noopener">http://doc.scrapy.org/en/latest/_static/selectors-sample1.html</a>。 开启 Scrapy shell，在命令行输入如下命令：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy shell http:<span class="regexp">//</span>doc.scrapy.org<span class="regexp">/en/</span>latest<span class="regexp">/_static/</span>selectors-sample1.html</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们就进入到 Scrapy shell 模式。这个过程其实是，Scrapy 发起了一次请求，请求的 URL 就是刚才命令行下输入的 URL，然后把一些可操作的变量传递给我们，如 request、response 等，如图 13-5 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-033928.jpg" alt=""> 图 13-5 Scrapy Shell 我们可以在命令行模式下输入命令调用对象的一些操作方法，回车之后实时显示结果。这与 Python 的命令行交互模式是类似的。 接下来，演示的实例都将页面的源码作为分析目标，页面源码如下所示：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">base</span> <span class="attr">href</span>=<span class="string">'http://example.com/'</span> /&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span>Example website<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">'images'</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">'image1.html'</span>&gt;</span>Name: My image 1 <span class="tag">&lt;<span class="name">br</span> /&gt;</span><span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">'image1_thumb.jpg'</span> /&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">'image2.html'</span>&gt;</span>Name: My image 2 <span class="tag">&lt;<span class="name">br</span> /&gt;</span><span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">'image2_thumb.jpg'</span> /&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">'image3.html'</span>&gt;</span>Name: My image 3 <span class="tag">&lt;<span class="name">br</span> /&gt;</span><span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">'image3_thumb.jpg'</span> /&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">'image4.html'</span>&gt;</span>Name: My image 4 <span class="tag">&lt;<span class="name">br</span> /&gt;</span><span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">'image4_thumb.jpg'</span> /&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">'image5.html'</span>&gt;</span>Name: My image 5 <span class="tag">&lt;<span class="name">br</span> /&gt;</span><span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">'image5_thumb.jpg'</span> /&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="3-XPath-选择器"><a href="#3-XPath-选择器" class="headerlink" title="3. XPath 选择器"></a>3. XPath 选择器</h3>
                  <p>进入 Scrapy shell 之后，我们将主要操作 response 这个变量来进行解析。因为我们解析的是 HTML 代码，Selector 将自动使用 HTML 语法来分析。 response 有一个属性 selector，我们调用 response.selector 返回的内容就相当于用 response 的 text 构造了一个 Selector 对象。通过这个 Selector 对象我们可以调用解析方法如 xpath()、css() 等，通过向方法传入 XPath 或 CSS 选择器参数就可以实现信息的提取。 我们用一个实例感受一下，如下所示：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; result = response.selector.xpath(<span class="comment">'//a')</span></span><br><span class="line">&gt;&gt;&gt; result</span><br><span class="line">[&lt;Selector xpath=<span class="comment">'//a' data='<span class="doctag">&lt;a href="image1.html"&gt;</span>Name: My image 1 <span class="doctag">&lt;'&gt;</span>,</span></span><br><span class="line"> &lt;Selector xpath=<span class="comment">'//a' data='<span class="doctag">&lt;a href="image2.html"&gt;</span>Name: My image 2 <span class="doctag">&lt;'&gt;</span>,</span></span><br><span class="line"> &lt;Selector xpath=<span class="comment">'//a' data='<span class="doctag">&lt;a href="image3.html"&gt;</span>Name: My image 3 <span class="doctag">&lt;'&gt;</span>,</span></span><br><span class="line"> &lt;Selector xpath=<span class="comment">'//a' data='<span class="doctag">&lt;a href="image4.html"&gt;</span>Name: My image 4 <span class="doctag">&lt;'&gt;</span>,</span></span><br><span class="line"> &lt;Selector xpath=<span class="comment">'//a' data='<span class="doctag">&lt;a href="image5.html"&gt;</span>Name: My image 5 <span class="doctag">&lt;'&gt;</span>]</span></span><br><span class="line">&gt;&gt;&gt; type(result)</span><br><span class="line">scrapy.selector.unified.SelectorList</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>打印结果的形式是 Selector 组成的列表，其实它是 SelectorList 类型，SelectorList 和 Selector 都可以继续调用 xpath() 和 css() 等方法来进一步提取数据。 在上面的例子中，我们提取了 a 节点。接下来，我们尝试继续调用 xpath() 方法来提取 a 节点内包含的 img 节点，如下所示：</p>
                  <figure class="highlight oxygene">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; <span class="keyword">result</span>.xpath(<span class="string">'./img'</span>)</span><br><span class="line">[&lt;<span class="keyword">Selector</span> xpath=<span class="string">'./img'</span> data=<span class="string">'&lt;img src="image1_thumb.jpg"&gt;'</span>&gt;,</span><br><span class="line"> &lt;<span class="keyword">Selector</span> xpath=<span class="string">'./img'</span> data=<span class="string">'&lt;img src="image2_thumb.jpg"&gt;'</span>&gt;,</span><br><span class="line"> &lt;<span class="keyword">Selector</span> xpath=<span class="string">'./img'</span> data=<span class="string">'&lt;img src="image3_thumb.jpg"&gt;'</span>&gt;,</span><br><span class="line"> &lt;<span class="keyword">Selector</span> xpath=<span class="string">'./img'</span> data=<span class="string">'&lt;img src="image4_thumb.jpg"&gt;'</span>&gt;,</span><br><span class="line"> &lt;<span class="keyword">Selector</span> xpath=<span class="string">'./img'</span> data=<span class="string">'&lt;img src="image5_thumb.jpg"&gt;'</span>&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们获得了 a 节点里面的所有 img 节点，结果为 5。 值得注意的是，选择器的最前方加 .（点），这代表提取元素内部的数据，如果没有加点，则代表从根节点开始提取。此处我们用了./img 的提取方式，则代表从 a 节点里进行提取。如果此处我们用 //img，则还是从 html 节点里进行提取。 我们刚才使用了 response.selector.xpath() 方法对数据进行了提取。Scrapy 提供了两个实用的快捷方法，response.xpath() 和 response.css()，它们二者的功能完全等同于 response.selector.xpath() 和 response.selector.css()。方便起见，后面我们统一直接调用 response 的 xpath() 和 css() 方法进行选择。 现在我们得到的是 SelectorList 类型的变量，该变量是由 Selector 对象组成的列表。我们可以用索引单独取出其中某个 Selector 元素，如下所示：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; result[0]</span><br><span class="line">&lt;Selector <span class="attribute">xpath</span>=<span class="string">'//a'</span> <span class="attribute">data</span>=<span class="string">'&lt;a href="image1.html"&gt;Name: My image 1 &lt;'</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以像操作列表一样操作这个 SelectorList。 但是现在获取的内容是 Selector 或者 SelectorList 类型，并不是真正的文本内容。那么具体的内容怎么提取呢？ 比如我们现在想提取出 a 节点元素，就可以利用 extract() 方法，如下所示：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; result.extract()</span><br><span class="line">[<span class="comment">'<span class="doctag">&lt;a href="image1.html"&gt;</span>Name: My image 1 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image1_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>', '<span class="doctag">&lt;a href="image2.html"&gt;</span>Name: My image 2 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image2_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>', '<span class="doctag">&lt;a href="image3.html"&gt;</span>Name: My image 3 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image3_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>', '<span class="doctag">&lt;a href="image4.html"&gt;</span>Name: My image 4 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image4_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>', '<span class="doctag">&lt;a href="image5.html"&gt;</span>Name: My image 5 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image5_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>']</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用了 extract() 方法，我们就可以把真实需要的内容获取下来。 我们还可以改写 XPath 表达式，来选取节点的内部文本和属性，如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.xpath(<span class="string">'//a/text()'</span>).extract()</span><br><span class="line">[<span class="string">'Name: My image 1 '</span>, <span class="string">'Name: My image 2 '</span>, <span class="string">'Name: My image 3 '</span>, <span class="string">'Name: My image 4 '</span>, <span class="string">'Name: My image 5 '</span>]</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; response.xpath(<span class="string">'//a/@href'</span>).extract()</span><br><span class="line">[<span class="string">'image1.html'</span>, <span class="string">'image2.html'</span>, <span class="string">'image3.html'</span>, <span class="string">'image4.html'</span>, <span class="string">'image5.html'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们只需要再加一层 /text() 就可以获取节点的内部文本，或者加一层 /@href 就可以获取节点的 href 属性。其中，@符号后面内容就是要获取的属性名称。 现在我们可以用一个规则把所有符合要求的节点都获取下来，返回的类型是列表类型。 但是这里有一个问题：如果符合要求的节点只有一个，那么返回的结果会是什么呢？我们再用一个实例来感受一下，如下所示：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; <span class="selector-tag">response</span><span class="selector-class">.xpath</span>(<span class="string">'//a[@href="image1.html"]/text()'</span>)<span class="selector-class">.extract</span>()</span><br><span class="line"><span class="selector-attr">['Name: My image 1 ']</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们用属性限制了匹配的范围，使 XPath 只可以匹配到一个元素。然后用 extract() 方法提取结果，其结果还是一个列表形式，其文本是列表的第一个元素。但很多情况下，我们其实想要的数据就是第一个元素内容，这里我们通过加一个索引来获取，如下所示： ```python&gt;&gt;&gt; response.xpath(‘//a[@href=”image1.html”]/text()’).extract()[0] ‘Name: My image 1 ‘</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"> 但是，这个写法很明显是有风险的。一旦 <span class="variable">XPath</span> 有问题，那么 <span class="function"><span class="title">extract</span>() 后的结果可能是一个空列表。如果我们再用索引来获取，那不就会可能导致数组越界吗？</span></span><br><span class="line"><span class="function"></span></span><br><span class="line"><span class="function">所以，另外一个方法可以专门提取单个元素，它叫作 <span class="title">extract_first</span>()。我们可以改写上面的例子如下所示：</span></span><br><span class="line"><span class="function"></span></span><br><span class="line"><span class="function">```<span class="variable">python</span></span></span><br><span class="line"><span class="function">&gt;&gt;&gt; <span class="variable">response.xpath</span>(<span class="string">'//a[@href="image1.html"]/text()'</span>).extract_first()</span></span><br><span class="line"><span class="string">'Name: My image 1 '</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样，我们直接利用 extract_first() 方法将匹配的第一个结果提取出来，同时我们也不用担心数组越界的问题。 另外我们也可以为 extract_first() 方法设置一个默认值参数，这样当 XPath 规则提取不到内容时会直接使用默认值。例如将 XPath 改成一个不存在的规则，重新执行代码，如下所示：</p>
                  <figure class="highlight mathematica">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.xpath('//a[@href=<span class="string">"image1"</span>]/text()').extract_first()&gt;&gt;&gt; response.xpath('//a[@href=<span class="string">"image1"</span>]/text()').extract_first('<span class="keyword">Default</span> <span class="keyword">Image</span>')</span><br><span class="line">'<span class="keyword">Default</span> <span class="keyword">Image</span>'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里，如果 XPath 匹配不到任何元素，调用 extract_first() 会返回空，也不会报错。 在第二行代码中，我们还传递了一个参数当作默认值，如 Default Image。这样如果 XPath 匹配不到结果的话，返回值会使用这个参数来代替，可以看到输出正是如此。 现在为止，我们了解了 Scrapy 中的 XPath 的相关用法，包括嵌套查询、提取内容、提取单个内容、获取文本和属性等。</p>
                  <h3 id="4-CSS-选择器"><a href="#4-CSS-选择器" class="headerlink" title="4. CSS 选择器"></a>4. CSS 选择器</h3>
                  <p>接下来，我们看看 CSS 选择器的用法。 Scrapy 的选择器同时还对接了 CSS 选择器，使用 response.css() 方法可以使用 CSS 选择器来选择对应的元素。 例如在上文我们选取了所有的 a 节点，那么 CSS 选择器同样可以做到，如下所示：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.css(<span class="comment">'a')</span></span><br><span class="line">[&lt;Selector xpath=<span class="comment">'descendant-or-self::a' data='<span class="doctag">&lt;a href="image1.html"&gt;</span>Name: My image 1 <span class="doctag">&lt;'&gt;</span>, </span></span><br><span class="line">&lt;Selector xpath=<span class="comment">'descendant-or-self::a' data='<span class="doctag">&lt;a href="image2.html"&gt;</span>Name: My image 2 <span class="doctag">&lt;'&gt;</span>, </span></span><br><span class="line">&lt;Selector xpath=<span class="comment">'descendant-or-self::a' data='<span class="doctag">&lt;a href="image3.html"&gt;</span>Name: My image 3 <span class="doctag">&lt;'&gt;</span>, </span></span><br><span class="line">&lt;Selector xpath=<span class="comment">'descendant-or-self::a' data='<span class="doctag">&lt;a href="image4.html"&gt;</span>Name: My image 4 <span class="doctag">&lt;'&gt;</span>, </span></span><br><span class="line">&lt;Selector xpath=<span class="comment">'descendant-or-self::a' data='<span class="doctag">&lt;a href="image5.html"&gt;</span>Name: My image 5 <span class="doctag">&lt;'&gt;</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>同样，调用 extract() 方法就可以提取出节点，如下所示： ```python&gt;&gt;&gt; response.css(‘a’).extract() <a href="image1.html">‘[Name: My image 1 <img src="image1_thumb.jpg" alt=""></a> ‘, ‘<a href="image2.html">Name: My image 2 <img src="image2_thumb.jpg" alt=""></a> ‘, ‘<a href="image3.html">Name: My image 3 <img src="image3_thumb.jpg" alt=""></a> ‘, ‘<a href="image4.html">Name: My image 4 <img src="image4_thumb.jpg" alt=""></a> ‘, ‘<a href="image5.html">Name: My image 5 <img src="image5_thumb.jpg" alt=""></a> ‘]</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"> 用法和 XPath 选择是完全一样的。</span><br><span class="line"></span><br><span class="line">另外，我们也可以进行属性选择和嵌套选择，如下所示：</span><br><span class="line"></span><br><span class="line">```python</span><br><span class="line">&gt;&gt;&gt; response.css(<span class="comment">'a[href="image1.html"]').extract()</span></span><br><span class="line">[<span class="comment">'<span class="doctag">&lt;a href="image1.html"&gt;</span>Name: My image 1 <span class="doctag">&lt;br&gt;</span><span class="doctag">&lt;img src="image1_thumb.jpg"&gt;</span><span class="doctag">&lt;/a&gt;</span>']</span></span><br><span class="line">&gt;&gt;&gt; response.css(<span class="comment">'a[href="image1.html"] img').extract()</span></span><br><span class="line">[<span class="comment">'<span class="doctag">&lt;img src="image1_thumb.jpg"&gt;</span>']</span></span><br><span class="line">​```这里用 [href=<span class="string">"image.html"</span>] 限定了 href 属性，可以看到匹配结果就只有一个了。另外如果想查找 a 节点内的 img 节点，只需要再加一个空格和 img 即可。选择器的写法和标准 CSS 选择器写法如出一辙。</span><br><span class="line"></span><br><span class="line">我们也可以使用 extract_first() 方法提取列表的第一个元素，如下所示：</span><br><span class="line"></span><br><span class="line">​```python</span><br><span class="line">&gt;&gt;&gt; response.css(<span class="comment">'a[href="image1.html"] img').extract_first()</span></span><br><span class="line"><span class="comment">'<span class="doctag">&lt;img src="image1_thumb.jpg"&gt;</span>'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来的两个用法不太一样。节点的内部文本和属性的获取是这样实现的，如下所示：</p>
                  <figure class="highlight rust">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.css(<span class="symbol">'a</span>[href=<span class="string">"image1.html"</span>]::text').extract_first()</span><br><span class="line"><span class="symbol">'Name</span>: My image <span class="number">1</span> '</span><br><span class="line">&gt;&gt;&gt; response.css(<span class="symbol">'a</span>[href=<span class="string">"image1.html"</span>] img::attr(src)').extract_first()</span><br><span class="line"><span class="symbol">'image1_thumb</span>.jpg'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>获取文本和属性需要用::text 和::attr() 的写法。而其他库如 Beautiful Soup 或 pyquery 都有单独的方法。 另外，CSS 选择器和 XPath 选择器一样可以嵌套选择。我们可以先用 XPath 选择器选中所有 a 节点，再利用 CSS 选择器选中 img 节点，再用 XPath 选择器获取属性。我们用一个实例来感受一下，如下所示：</p>
                  <figure class="highlight livescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">\&gt;&gt;&gt;</span> response.xpath(<span class="string">'//a'</span>).css(<span class="string">'img'</span>).xpath(<span class="string">'@src'</span>).extract()</span><br><span class="line">[<span class="string">'image1_thumb.jpg'</span>, <span class="string">'image2_thumb.jpg'</span>, <span class="string">'image3_thumb.jpg'</span>, <span class="string">'image4_thumb.jpg'</span>, <span class="string">'image5_thumb.jpg'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们成功获取了所有 img 节点的 src 属性。 因此，我们可以随意使用 xpath() 和 css() 方法二者自由组合实现嵌套查询，二者是完全兼容的。</p>
                  <h3 id="5-正则匹配"><a href="#5-正则匹配" class="headerlink" title="5. 正则匹配"></a>5. 正则匹配</h3>
                  <p>Scrapy 的选择器还支持正则匹配。比如，在示例的 a 节点中的文本类似于 Name: My image 1，现在我们只想把 Name: 后面的内容提取出来，这时就可以借助 re() 方法，实现如下：</p>
                  <figure class="highlight livescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">\&gt;&gt;&gt;</span> response.xpath(<span class="string">'//a/text()'</span>).re(<span class="string">'Name:s(.*)'</span>)</span><br><span class="line">[<span class="string">'My image 1 '</span>, <span class="string">'My image 2 '</span>, <span class="string">'My image 3 '</span>, <span class="string">'My image 4 '</span>, <span class="string">'My image 5 '</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们给 re() 方法传了一个正则表达式，其中 (.*) 就是要匹配的内容，输出的结果就是正则表达式匹配的分组，结果会依次输出。 如果同时存在两个分组，那么结果依然会被按序输出，如下所示：</p>
                  <figure class="highlight livescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">\&gt;&gt;&gt;</span> response.xpath(<span class="string">'//a/text()'</span>).re(<span class="string">'(.*?):s(.*)'</span>)</span><br><span class="line">[<span class="string">'Name'</span>, <span class="string">'My image 1 '</span>, <span class="string">'Name'</span>, <span class="string">'My image 2 '</span>, <span class="string">'Name'</span>, <span class="string">'My image 3 '</span>, <span class="string">'Name'</span>, <span class="string">'My image 4 '</span>, <span class="string">'Name'</span>, <span class="string">'My image 5 '</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>类似 extract_first() 方法，re_first() 方法可以选取列表的第一个元素，用法如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.xpath(<span class="string">'//a/text()'</span>).re_first(<span class="string">'(.*?):s(.*)'</span>)</span><br><span class="line"><span class="string">'Name'</span></span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; response.xpath(<span class="string">'//a/text()'</span>).re_first(<span class="string">'Name:s(.*)'</span>)</span><br><span class="line"><span class="string">'My image 1 '</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不论正则匹配了几个分组，结果都会等于列表的第一个元素。 值得注意的是，response 对象不能直接调用 re() 和 re_first() 方法。如果想要对全文进行正则匹配，可以先调用 xpath() 方法再正则匹配，如下所示：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">\&gt;&gt;&gt; response.re(<span class="string">'Name:s(.*)'</span>)</span><br><span class="line">Traceback (most recent <span class="keyword">call</span> last):</span><br><span class="line">  File "&lt;console&gt;", <span class="type">line</span> <span class="number">1</span>, <span class="keyword">in</span> &lt;module&gt;</span><br><span class="line">AttributeError: <span class="string">'HtmlResponse'</span> <span class="keyword">object</span> has <span class="keyword">no</span> <span class="keyword">attribute</span> <span class="string">'re'</span></span><br><span class="line">&gt;&gt;&gt; response.xpath(<span class="string">'.'</span>).re(<span class="string">'Name:s(.*)&lt;br&gt;'</span>)</span><br><span class="line">[<span class="string">'My image 1 '</span>, <span class="string">'My image 2 '</span>, <span class="string">'My image 3 '</span>, <span class="string">'My image 4 '</span>, <span class="string">'My image 5 '</span>]</span><br><span class="line">&gt;&gt;&gt; response.xpath(<span class="string">'.'</span>).re_first(<span class="string">'Name:s(.*)&lt;br&gt;'</span>)</span><br><span class="line"><span class="string">'My image 1 '</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过上面的例子，我们可以看到，直接调用 re() 方法会提示没有 re 属性。但是这里首先调用了 xpath(‘.’) 选中全文，然后调用 re() 和 re_first() 方法，就可以进行正则匹配了。</p>
                  <h3 id="6-结语"><a href="#6-结语" class="headerlink" title="6. 结语"></a>6. 结语</h3>
                  <p>以上内容便是 Scrapy 选择器的用法，它包括两个常用选择器和正则匹配功能。熟练掌握 XPath 语法、CSS 选择器语法、正则表达式语法可以大大提高数据提取效率。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-03 09:31:21" itemprop="dateCreated datePublished" datetime="2019-12-03T09:31:21+08:00">2019-12-03</time>
                </span>
                <span id="/8350.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.3–Selector 的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8337.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/8337.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.2-Scrapy 入门</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-2-Scrapy-入门"><a href="#13-2-Scrapy-入门" class="headerlink" title="13.2 Scrapy 入门"></a>13.2 Scrapy 入门</h1>
                  <p>接下来介绍一个简单的项目，完成一遍 Scrapy 抓取流程。通过这个过程，我们可以对 Scrapy 的基本用法和原理有大体了解。</p>
                  <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                  <p>本节要完成的任务如下。</p>
                  <ul>
                    <li>创建一个 Scrapy 项目。</li>
                    <li>创建一个 Spider 来抓取站点和处理数据。</li>
                    <li>通过命令行将抓取的内容导出。</li>
                    <li>将抓取的内容保存到 MongoDB 数据库。</li>
                  </ul>
                  <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                  <p>我们需要安装好 Scrapy 框架、MongoDB 和 PyMongo 库。如果尚未安装，请参照上一节的安装说明。</p>
                  <h3 id="3-创建项目"><a href="#3-创建项目" class="headerlink" title="3. 创建项目"></a>3. 创建项目</h3>
                  <p>创建一个 Scrapy 项目，项目文件可以直接用 scrapy 命令生成，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy startproject tutorial</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个命令可以在任意文件夹运行。如果提示权限问题，可以加 sudo 运行该命令。这个命令将会创建一个名为 tutorial 的文件夹，文件夹结构如下所示：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy.cfg     <span class="comment"># Scrapy 部署时的配置文件</span></span><br><span class="line">tutorial         <span class="comment"># 项目的模块，引入的时候需要从这里引入</span></span><br><span class="line">    __init__.py    </span><br><span class="line">    <span class="keyword">items</span>.py     <span class="comment"># Items 的定义，定义爬取的数据结构</span></span><br><span class="line">    middlewares.py   <span class="comment"># Middlewares 的定义，定义爬取时的中间件</span></span><br><span class="line">    pipelines.py       <span class="comment"># Pipelines 的定义，定义数据管道</span></span><br><span class="line">    settings.py       <span class="comment"># 配置文件</span></span><br><span class="line">    spiders         <span class="comment"># 放置 Spiders 的文件夹</span></span><br><span class="line">    __init__.py</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="4-创建-Spider"><a href="#4-创建-Spider" class="headerlink" title="4. 创建 Spider"></a>4. 创建 Spider</h3>
                  <p>Spider 是自己定义的类，Scrapy 用它来从网页里抓取内容，并解析抓取的结果。不过这个类必须继承 Scrapy 提供的 Spider 类 scrapy.Spider，还要定义 Spider 的名称和起始请求，以及怎样处理爬取后的结果的方法。 也可以使用命令行创建一个 Spider。比如要生成 Quotes 这个 Spider，可以执行如下命令：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">cd</span> <span class="string">tutorial</span></span><br><span class="line"><span class="attr">scrapy</span> <span class="string">genspider quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>进入刚才创建的 tutorial 文件夹，然后执行 genspider 命令。第一个参数是 Spider 的名称，第二个参数是网站域名。执行完毕之后，spiders 文件夹中多了一个 quotes.py，它就是刚刚创建的 Spider，内容如下所示：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">QuotesSpider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line">    name = <span class="string">"quotes"</span></span><br><span class="line">    allowed_domains = [<span class="string">"quotes.toscrape.com"</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://quotes.toscrape.com/'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        <span class="keyword">pass</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里有三个属性 ——name、allowed_domains 和 start_urls，还有一个方法 parse。</p>
                  <ul>
                    <li>name，它是每个项目唯一的名字，用来区分不同的 Spider。</li>
                    <li>allowed_domains，它是允许爬取的域名，如果初始或后续的请求链接不是这个域名下的，则请求链接会被过滤掉。</li>
                    <li>start_urls，它包含了 Spider 在启动时爬取的 url 列表，初始请求是由它来定义的。</li>
                    <li>parse，它是 Spider 的一个方法。默认情况下，被调用时 start_urls 里面的链接构成的请求完成下载执行后，返回的响应就会作为唯一的参数传递给这个函数。该方法负责解析返回的响应、提取数据或者进一步生成要处理的请求。</li>
                  </ul>
                  <h3 id="5-创建-Item"><a href="#5-创建-Item" class="headerlink" title="5. 创建 Item"></a>5. 创建 Item</h3>
                  <p>Item 是保存爬取数据的容器，它的使用方法和字典类似。不过，相比字典，Item 多了额外的保护机制，可以避免拼写错误或者定义字段错误。 创建 Item 需要继承 scrapy.Item 类，并且定义类型为 scrapy.Field 的字段。观察目标网站，我们可以获取到的内容有 text、author、tags。 定义 Item，此时将 items.py 修改如下：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">QuoteItem</span>(<span class="title">scrapy</span>.<span class="type">Item</span>):</span></span><br><span class="line"><span class="class"></span></span><br><span class="line"><span class="class">    text = scrapy.<span class="type">Field</span>()</span></span><br><span class="line"><span class="class">    author = scrapy.<span class="type">Field</span>()</span></span><br><span class="line"><span class="class">    tags = scrapy.<span class="type">Field</span>()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了三个字段，将类的名称修改为 QuoteItem，接下来爬取时我们会使用到这个 Item。</p>
                  <h3 id="6-解析-Response"><a href="#6-解析-Response" class="headerlink" title="6. 解析 Response"></a>6. 解析 Response</h3>
                  <p>前面我们看到，parse() 方法的参数 response 是 start_urls 里面的链接爬取后的结果。所以在 parse() 方法中，我们可以直接对 response 变量包含的内容进行解析，比如浏览请求结果的网页源代码，或者进一步分析源代码内容，或者找出结果中的链接而得到下一个请求。 我们可以看到网页中既有我们想要的结果，又有下一页的链接，这两部分内容我们都要进行处理。 首先看看网页结构，如图 13-2 所示。每一页都有多个 class 为 quote 的区块，每个区块内都包含 text、author、tags。那么我们先找出所有的 quote，然后提取每一个 quote 中的内容。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-033901.jpg" alt=""> 图 13-2 页面结构 提取的方式可以是 CSS 选择器或 XPath 选择器。在这里我们使用 CSS 选择器进行选择，parse() 方法的改写如下所示：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse(self, response):</span><br><span class="line">    quotes = response.css('.<span class="literal">quote</span>')</span><br><span class="line">    <span class="keyword">for</span> <span class="literal">quote</span> <span class="keyword">in</span> quotes:</span><br><span class="line">        <span class="built_in">text</span> = <span class="literal">quote</span>.css('.<span class="built_in">text</span>::<span class="built_in">text</span>').extract_first()</span><br><span class="line">        author = <span class="literal">quote</span>.css('.author::<span class="built_in">text</span>').extract_first()</span><br><span class="line">        tags = <span class="literal">quote</span>.css('.tags .tag::<span class="built_in">text</span>').extract()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先利用选择器选取所有的 quote，并将其赋值为 quotes 变量，然后利用 for 循环对每个 quote 遍历，解析每个 quote 的内容。 对 text 来说，观察到它的 class 为 text，所以可以用.text 选择器来选取，这个结果实际上是整个带有标签的节点，要获取它的正文内容，可以加::text 来获取。这时的结果是长度为 1 的列表，所以还需要用 extract_first() 方法来获取第一个元素。而对于 tags 来说，由于我们要获取所有的标签，所以用 extract() 方法获取整个列表即可。 以第一个 quote 的结果为例，各个选择方法及结果的说明如下内容。 源码如下：</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"quote"</span> itemscope=<span class="string">""</span>itemtype=<span class="string">"http://schema.org/CreativeWork"</span>&gt;</span><br><span class="line">        &lt;span <span class="class"><span class="keyword">class</span></span>=<span class="string">"text"</span> itemprop=<span class="string">"text"</span>&gt;“The world <span class="keyword">as</span> we have created it is a process <span class="keyword">of</span> our thinking. It cannot be changed without changing our thinking.”&lt;<span class="regexp">/span&gt;</span></span><br><span class="line"><span class="regexp">        &lt;span&gt;by &lt;small class="author" itemprop="author"&gt;Albert Einstein&lt;/</span>small&gt;</span><br><span class="line">        &lt;a href=<span class="string">"/author/Albert-Einstein"</span>&gt;(about)&lt;<span class="regexp">/a&gt;</span></span><br><span class="line"><span class="regexp">        &lt;/</span>span&gt;</span><br><span class="line">        &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"tags"</span>&gt;</span><br><span class="line">            Tags:</span><br><span class="line">            &lt;meta <span class="class"><span class="keyword">class</span></span>=<span class="string">"keywords"</span> itemprop=<span class="string">"keywords"</span> content=<span class="string">"change,deep-thoughts,thinking,world"</span>&gt; </span><br><span class="line">            &lt;a <span class="class"><span class="keyword">class</span></span>=<span class="string">"tag"</span> href=<span class="string">"/tag/change/page/1/"</span>&gt;change&lt;<span class="regexp">/a&gt;</span></span><br><span class="line"><span class="regexp">            &lt;a class="tag" href="/</span>tag/deep-thoughts/page/<span class="number">1</span>/<span class="string">"&gt;deep-thoughts&lt;/a&gt;</span></span><br><span class="line"><span class="string">            &lt;a class="</span>tag<span class="string">" href="</span>/tag/thinking/page/<span class="number">1</span>/<span class="string">"&gt;thinking&lt;/a&gt;</span></span><br><span class="line"><span class="string">            &lt;a class="</span>tag<span class="string">" href="</span>/tag/world/page/<span class="number">1</span>/<span class="string">"&gt;world&lt;/a&gt;</span></span><br><span class="line"><span class="string">        &lt;/div&gt;</span></span><br><span class="line"><span class="string">    &lt;/div&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不同选择器的返回结果如下。</p>
                  <h4 id="quote-css-‘-text’"><a href="#quote-css-‘-text’" class="headerlink" title="quote.css(‘.text’)"></a>quote.css(‘.text’)</h4>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Selector xpath="descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' text ')]"data='&lt;span class="text"itemprop="text"&gt;“The '&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="quote-css-‘-text-text’"><a href="#quote-css-‘-text-text’" class="headerlink" title="quote.css(‘.text::text’)"></a>quote.css(‘.text::text’)</h4>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Selector xpath="descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' text ')]/text()"data='“The world as we have created it is a pr'&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="quote-css-‘-text’-extract"><a href="#quote-css-‘-text’-extract" class="headerlink" title="quote.css(‘.text’).extract()"></a>quote.css(‘.text’).extract()</h4>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['&lt;span <span class="built_in">class</span>=<span class="string">"text"</span>itemprop=<span class="string">"text"</span>&gt;“The world <span class="keyword">as</span> we have created <span class="keyword">it</span> <span class="keyword">is</span> a process <span class="keyword">of</span> our thinking. It cannot be changed <span class="keyword">without</span> changing our thinking.”&lt;/span&gt;']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="quote-css-‘-text-text’-extract"><a href="#quote-css-‘-text-text’-extract" class="headerlink" title="quote.css(‘.text::text’).extract()"></a>quote.css(‘.text::text’).extract()</h4>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['“The world <span class="keyword">as</span> we have created <span class="keyword">it</span> <span class="keyword">is</span> a process <span class="keyword">of</span> our thinking. It cannot be changed <span class="keyword">without</span> changing our thinking.”']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="quote-css-‘-text-text’-extract-first"><a href="#quote-css-‘-text-text’-extract-first" class="headerlink" title="quote.css(‘.text::text’).extract_first()"></a>quote.css(‘.text::text’).extract_first()</h4>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">“The world <span class="keyword">as</span> we have created <span class="keyword">it</span> is <span class="keyword">a</span> <span class="built_in">process</span> <span class="keyword">of</span> our thinking. It cannot be changed <span class="keyword">without</span> changing our thinking.”</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以，对于 text，获取结果的第一个元素即可，所以使用 extract_first() 方法，对于 tags，要获取所有结果组成的列表，所以使用 extract() 方法。</p>
                  <h3 id="7-使用-Item"><a href="#7-使用-Item" class="headerlink" title="7. 使用 Item"></a>7. 使用 Item</h3>
                  <p>上文定义了 Item，接下来就要使用它了。Item 可以理解为一个字典，不过在声明的时候需要实例化。然后依次用刚才解析的结果赋值 Item 的每一个字段，最后将 Item 返回即可。 QuotesSpider 的改写如下所示：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import scrapy</span><br><span class="line"><span class="built_in">from</span> tutorial.<span class="keyword">items</span> import QuoteItem</span><br><span class="line"></span><br><span class="line">class QuotesSpider(scrapy.Spider):</span><br><span class="line">    name = <span class="string">"quotes"</span></span><br><span class="line">    allowed_domains = [<span class="string">"quotes.toscrape.com"</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://quotes.toscrape.com/'</span>]</span><br><span class="line"></span><br><span class="line">    def parse(self, response):</span><br><span class="line">        quotes = response.css(<span class="string">'.quote'</span>)</span><br><span class="line">        <span class="keyword">for</span> <span class="literal">quote</span> <span class="keyword">in</span> quotes:</span><br><span class="line">            <span class="keyword">item</span> = QuoteItem()</span><br><span class="line">            <span class="keyword">item</span>[<span class="string">'text'</span>] = <span class="literal">quote</span>.css(<span class="string">'.text::text'</span>).extract_first()</span><br><span class="line">            <span class="keyword">item</span>[<span class="string">'author'</span>] = <span class="literal">quote</span>.css(<span class="string">'.author::text'</span>).extract_first()</span><br><span class="line">            <span class="keyword">item</span>[<span class="string">'tags'</span>] = <span class="literal">quote</span>.css(<span class="string">'.tags .tag::text'</span>).extract()</span><br><span class="line">            yield <span class="keyword">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如此一来，首页的所有内容被解析出来，并被赋值成了一个个 QuoteItem。</p>
                  <h3 id="8-后续-Request"><a href="#8-后续-Request" class="headerlink" title="8. 后续 Request"></a>8. 后续 Request</h3>
                  <p>上面的操作实现了从初始页面抓取内容。那么，下一页的内容该如何抓取？这就需要我们从当前页面中找到信息来生成下一个请求，然后在下一个请求的页面里找到信息再构造再下一个请求。这样循环往复迭代，从而实现整站的爬取。 将刚才的页面拉到最底部，如图 13-3 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-033909.jpg" alt=""> 图 13-3 页面底部 有一个 Next 按钮，查看一下源代码，可以发现它的链接是 /page/2/，实际上全链接就是：<a href="http://quotes.toscrape.com/page/2" target="_blank" rel="noopener">http://quotes.toscrape.com/page/2</a>，通过这个链接我们就可以构造下一个请求。 构造请求时需要用到 scrapy.Request。这里我们传递两个参数 ——url 和 callback，这两个参数的说明如下。</p>
                  <ul>
                    <li>url：它是请求链接。</li>
                    <li>callback：它是回调函数。当指定了该回调函数的请求完成之后，获取到响应，引擎会将该响应作为参数传递给这个回调函数。回调函数进行解析或生成下一个请求，回调函数如上文的 parse() 所示。</li>
                  </ul>
                  <p>由于 parse() 就是解析 text、author、tags 的方法，而下一页的结构和刚才已经解析的页面结构是一样的，所以我们可以再次使用 parse() 方法来做页面解析。 接下来我们要做的就是利用选择器得到下一页链接并生成请求，在 parse() 方法后追加如下的代码：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">next = response.css('.pager .next a::attr(href)').extract<span class="constructor">_first()</span></span><br><span class="line">url = response.urljoin(next)</span><br><span class="line">yield scrapy.<span class="constructor">Request(<span class="params">url</span>=<span class="params">url</span>, <span class="params">callback</span>=<span class="params">self</span>.<span class="params">parse</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一句代码首先通过 CSS 选择器获取下一个页面的链接，即要获取 a 超链接中的 href 属性。这里用到了::attr(href) 操作。然后再调用 extract_first() 方法获取内容。 第二句代码调用了 urljoin() 方法，urljoin() 方法可以将相对 URL 构造成一个绝对的 URL。例如，获取到的下一页地址是 /page/2，urljoin() 方法处理后得到的结果就是：<a href="http://quotes.toscrape.com/page/2/" target="_blank" rel="noopener">http://quotes.toscrape.com/page/2/</a>。 第三句代码通过 url 和 callback 变量构造了一个新的请求，回调函数 callback 依然使用 parse() 方法。这个请求完成后，响应会重新经过 parse 方法处理，得到第二页的解析结果，然后生成第二页的下一页，也就是第三页的请求。这样爬虫就进入了一个循环，直到最后一页。 通过几行代码，我们就轻松实现了一个抓取循环，将每个页面的结果抓取下来了。 现在，改写之后的整个 Spider 类如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import scrapy</span><br><span class="line">from tutorial.items import QuoteItem</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">QuotesSpider</span>(<span class="title">scrapy</span>.<span class="title">Spider</span>):</span></span><br><span class="line">    name = <span class="string">"quotes"</span></span><br><span class="line">    allowed_domains = [<span class="string">"quotes.toscrape.com"</span>]</span><br><span class="line">    start_urls = [<span class="string">'http://quotes.toscrape.com/'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        quotes = response.css(<span class="string">'.quote'</span>)</span><br><span class="line">        <span class="keyword">for</span> quote <span class="keyword">in</span> <span class="symbol">quotes:</span></span><br><span class="line">            item = QuoteItem()</span><br><span class="line">            item[<span class="string">'text'</span>] = quote.css(<span class="string">'.text::text'</span>).extract_first()</span><br><span class="line">            item[<span class="string">'author'</span>] = quote.css(<span class="string">'.author::text'</span>).extract_first()</span><br><span class="line">            item[<span class="string">'tags'</span>] = quote.css(<span class="string">'.tags .tag::text'</span>).extract()</span><br><span class="line">            <span class="keyword">yield</span> item</span><br><span class="line"></span><br><span class="line">        <span class="keyword">next</span> = response.css(<span class="string">'.pager .next a::attr("href")'</span>).extract_first()</span><br><span class="line">        url = response.urljoin(<span class="keyword">next</span>)</span><br><span class="line">        <span class="keyword">yield</span> scrapy.Request(url=url, callback=<span class="keyword">self</span>.parse)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="9-运行"><a href="#9-运行" class="headerlink" title="9. 运行"></a>9. 运行</h3>
                  <p>接下来，进入目录，运行如下命令：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>就可以看到 Scrapy 的运行结果了。</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.utils.log] INFO: Scrapy <span class="number">1.3</span><span class="number">.0</span> started (bot: tutorial)</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.utils.log] INFO: Overridden settings: &#123;<span class="string">'NEWSPIDER_MODULE'</span>: <span class="string">'tutorial.spiders'</span>, <span class="string">'SPIDER_MODULES'</span>: [<span class="string">'tutorial.spiders'</span>], <span class="string">'ROBOTSTXT_OBEY'</span>: <span class="literal">True</span>, <span class="string">'BOT_NAME'</span>: <span class="string">'tutorial'</span>&#125;</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.middleware] INFO: Enabled extensions:</span><br><span class="line">[<span class="string">'scrapy.extensions.logstats.LogStats'</span>,</span><br><span class="line"> <span class="string">'scrapy.extensions.telnet.TelnetConsole'</span>,</span><br><span class="line"> <span class="string">'scrapy.extensions.corestats.CoreStats'</span>]</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.middleware] INFO: Enabled downloader middlewares:</span><br><span class="line">[<span class="string">'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.retry.RetryMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.cookies.CookiesMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.downloadermiddlewares.stats.DownloaderStats'</span>]</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.middleware] INFO: Enabled spider middlewares:</span><br><span class="line">[<span class="string">'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.spidermiddlewares.offsite.OffsiteMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.spidermiddlewares.referer.RefererMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware'</span>,</span><br><span class="line"> <span class="string">'scrapy.spidermiddlewares.depth.DepthMiddleware'</span>]</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.middleware] INFO: Enabled item pipelines:</span><br><span class="line">[]</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.core.engine] INFO: Spider opened</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.extensions.logstats] INFO: Crawled <span class="number">0</span> pages (at <span class="number">0</span> pages/min), scraped <span class="number">0</span> items (at <span class="number">0</span> items/min)</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">20</span> [scrapy.extensions.telnet] DEBUG: Telnet console listening on <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span>:<span class="number">6023</span></span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">21</span> [scrapy.core.engine] DEBUG: Crawled (<span class="number">404</span>) &lt;GET http://quotes.toscrape.com/robots.txt&gt; (referer: <span class="literal">None</span>)</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">21</span> [scrapy.core.engine] DEBUG: Crawled (<span class="number">200</span>) &lt;GET http://quotes.toscrape.com/&gt; (referer: <span class="literal">None</span>)</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">21</span> [scrapy.core.scraper] DEBUG: Scraped <span class="keyword">from</span> &lt;<span class="number">200</span> http://quotes.toscrape.com/&gt;</span><br><span class="line">&#123;<span class="string">'author'</span>: <span class="string">u'Albert Einstein'</span>,</span><br><span class="line"> <span class="string">'tags'</span>: [<span class="string">u'change'</span>, <span class="string">u'deep-thoughts'</span>, <span class="string">u'thinking'</span>, <span class="string">u'world'</span>],</span><br><span class="line"> <span class="string">'text'</span>: <span class="string">u'u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.u201d'</span>&#125;</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">21</span> [scrapy.core.scraper] DEBUG: Scraped <span class="keyword">from</span> &lt;<span class="number">200</span> http://quotes.toscrape.com/&gt;</span><br><span class="line">&#123;<span class="string">'author'</span>: <span class="string">u'J.K. Rowling'</span>,</span><br><span class="line"> <span class="string">'tags'</span>: [<span class="string">u'abilities'</span>, <span class="string">u'choices'</span>],</span><br><span class="line"> <span class="string">'text'</span>: <span class="string">u'u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.u201d'</span>&#125;</span><br><span class="line">...</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">27</span> [scrapy.core.engine] INFO: Closing spider (finished)</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">27</span> [scrapy.statscollectors] INFO: Dumping Scrapy stats:</span><br><span class="line">&#123;<span class="string">'downloader/request_bytes'</span>: <span class="number">2859</span>,</span><br><span class="line"> <span class="string">'downloader/request_count'</span>: <span class="number">11</span>,</span><br><span class="line"> <span class="string">'downloader/request_method_count/GET'</span>: <span class="number">11</span>,</span><br><span class="line"> <span class="string">'downloader/response_bytes'</span>: <span class="number">24871</span>,</span><br><span class="line"> <span class="string">'downloader/response_count'</span>: <span class="number">11</span>,</span><br><span class="line"> <span class="string">'downloader/response_status_count/200'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'downloader/response_status_count/404'</span>: <span class="number">1</span>,</span><br><span class="line"> <span class="string">'dupefilter/filtered'</span>: <span class="number">1</span>,</span><br><span class="line"> <span class="string">'finish_reason'</span>: <span class="string">'finished'</span>,</span><br><span class="line"> <span class="string">'finish_time'</span>: datetime.datetime(<span class="number">2017</span>, <span class="number">2</span>, <span class="number">19</span>, <span class="number">5</span>, <span class="number">37</span>, <span class="number">27</span>, <span class="number">227438</span>),</span><br><span class="line"> <span class="string">'item_scraped_count'</span>: <span class="number">100</span>,</span><br><span class="line"> <span class="string">'log_count/DEBUG'</span>: <span class="number">113</span>,</span><br><span class="line"> <span class="string">'log_count/INFO'</span>: <span class="number">7</span>,</span><br><span class="line"> <span class="string">'request_depth_max'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'response_received_count'</span>: <span class="number">11</span>,</span><br><span class="line"> <span class="string">'scheduler/dequeued'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'scheduler/dequeued/memory'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'scheduler/enqueued'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'scheduler/enqueued/memory'</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">'start_time'</span>: datetime.datetime(<span class="number">2017</span>, <span class="number">2</span>, <span class="number">19</span>, <span class="number">5</span>, <span class="number">37</span>, <span class="number">20</span>, <span class="number">321557</span>)&#125;</span><br><span class="line"><span class="number">2017</span><span class="number">-02</span><span class="number">-19</span> <span class="number">13</span>:<span class="number">37</span>:<span class="number">27</span> [scrapy.core.engine] INFO: Spider closed (finished)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里只是部分运行结果，中间一些抓取结果已省略。 首先，Scrapy 输出了当前的版本号以及正在启动的项目名称。接着输出了当前 settings.py 中一些重写后的配置。然后输出了当前所应用的 Middlewares 和 Pipelines。Middlewares 默认是启用的，可以在 settings.py 中修改。Pipelines 默认是空，同样也可以在 settings.py 中配置。后面会对它们进行讲解。 接下来就是输出各个页面的抓取结果了，可以看到爬虫一边解析，一边翻页，直至将所有内容抓取完毕，然后终止。 最后，Scrapy 输出了整个抓取过程的统计信息，如请求的字节数、请求次数、响应次数、完成原因等。 整个 Scrapy 程序成功运行。我们通过非常简单的代码就完成了一个网站内容的爬取，这样相比之前一点点写程序简洁很多。</p>
                  <h3 id="10-保存到文件"><a href="#10-保存到文件" class="headerlink" title="10. 保存到文件"></a>10. 保存到文件</h3>
                  <p>运行完 Scrapy 后，我们只在控制台看到了输出结果。如果想保存结果该怎么办呢？ 要完成这个任务其实不需要任何额外的代码，Scrapy 提供的 Feed Exports 可以轻松将抓取结果输出。例如，我们想将上面的结果保存成 JSON 文件，可以执行如下命令：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.json</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令运行后，项目内多了一个 quotes.json 文件，文件包含了刚才抓取的所有内容，内容是 JSON 格式。 另外我们还可以每一个 Item 输出一行 JSON，输出后缀为 jl，为 jsonline 的缩写，命令如下所示：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.jl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.jsonlines</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出格式还支持很多种，例如 csv、xml、pickle、marshal 等，还支持 ftp、s3 等远程输出，另外还可以通过自定义 ItemExporter 来实现其他的输出。 例如，下面命令对应的输出分别为 csv、xml、pickle、marshal 格式以及 ftp 远程输出：</p>
                  <figure class="highlight scss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.csv</span></span><br><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.xml</span></span><br><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.pickle</span></span><br><span class="line">scrapy crawl <span class="attribute">quotes</span> -o <span class="attribute">quotes</span><span class="selector-class">.marshal</span></span><br><span class="line">scrapy crawl <span class="attribute">quotes</span> -o ftp://user:pass@ftp.example.com/path/to/quotes.csv</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中，ftp 输出需要正确配置用户名、密码、地址、输出路径，否则会报错。 通过 Scrapy 提供的 Feed Exports，我们可以轻松地输出抓取结果到文件。对于一些小型项目来说，这应该足够了。不过如果想要更复杂的输出，如输出到数据库等，我们可以使用 Item Pileline 来完成。</p>
                  <h3 id="11-使用-Item-Pipeline"><a href="#11-使用-Item-Pipeline" class="headerlink" title="11. 使用 Item Pipeline"></a>11. 使用 Item Pipeline</h3>
                  <p>如果想进行更复杂的操作，如将结果保存到 MongoDB 数据库，或者筛选某些有用的 Item，则我们可以定义 Item Pipeline 来实现。 Item Pipeline 为项目管道。当 Item 生成后，它会自动被送到 Item Pipeline 进行处理，我们常用 Item Pipeline 来做如下操作。</p>
                  <ul>
                    <li>清洗 HTML 数据</li>
                    <li>验证爬取数据，检查爬取字段</li>
                    <li>查重并丢弃重复内容</li>
                    <li>将爬取结果储存到数据库</li>
                  </ul>
                  <p>要实现 Item Pipeline 很简单，只需要定义一个类并实现 process_item() 方法即可。启用 Item Pipeline 后，Item Pipeline 会自动调用这个方法。process_item() 方法必须返回包含数据的字典或 Item 对象，或者抛出 DropItem 异常。 process_item() 方法有两个参数。一个参数是 item，每次 Spider 生成的 Item 都会作为参数传递过来。另一个参数是 spider，就是 Spider 的实例。 接下来，我们实现一个 Item Pipeline，筛掉 text 长度大于 50 的 Item，并将结果保存到 MongoDB。 修改项目里的 pipelines.py 文件，之前用命令行自动生成的文件内容可以删掉，增加一个 TextPipeline 类，内容如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapy.exceptions import DropItem</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">TextPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.limit = <span class="number">50</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> item[<span class="string">'text'</span>]<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">if</span> len(item[<span class="string">'text'</span>]) &gt; <span class="keyword">self</span>.<span class="symbol">limit:</span></span><br><span class="line">                item[<span class="string">'text'</span>] = item[<span class="string">'text'</span>][<span class="number">0</span><span class="symbol">:self</span>.limit].rstrip() + <span class="string">'...'</span></span><br><span class="line">            <span class="keyword">return</span> item</span><br><span class="line">        <span class="symbol">else:</span></span><br><span class="line">            <span class="keyword">return</span> DropItem(<span class="string">'Missing Text'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这段代码在构造方法里定义了限制长度为 50，实现了 process_item() 方法，其参数是 item 和 spider。首先该方法判断 item 的 text 属性是否存在，如果不存在，则抛出 DropItem 异常；如果存在，再判断长度是否大于 50，如果大于，那就截断然后拼接省略号，再将 item 返回即可。 接下来，我们将处理后的 item 存入 MongoDB，定义另外一个 Pipeline。同样在 pipelines.py 中，我们实现另一个类 MongoPipeline，内容如下所示：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymongo</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>),</span><br><span class="line">            mongo_db=crawler.settings.get(<span class="string">'MONGO_DB'</span>)</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        name = item.__class_<span class="number">_</span>.__name_<span class="number">_</span></span><br><span class="line">        <span class="keyword">self</span>.db[name].insert(dict(item))</span><br><span class="line">        <span class="keyword">return</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>MongoPipeline 类实现了 API 定义的另外几个方法。</p>
                  <ul>
                    <li>from_crawler，这是一个类方法，用 @classmethod 标识，是一种依赖注入的方式，方法的参数就是 crawler，通过 crawler 这个我们可以拿到全局配置的每个配置信息，在全局配置 settings.py 中我们可以定义 MONGO_URI 和 MONGO_DB 来指定 MongoDB 连接需要的地址和数据库名称，拿到配置信息之后返回类对象即可。所以这个方法的定义主要是用来获取 settings.py 中的配置的。</li>
                    <li>open_spider，当 Spider 被开启时，这个方法被调用。在这里主要进行了一些初始化操作。</li>
                    <li>close_spider，当 Spider 被关闭时，这个方法会调用，在这里将数据库连接关闭。</li>
                  </ul>
                  <p>最主要的 process_item() 方法则执行了数据插入操作。 定义好 TextPipeline 和 MongoPipeline 这两个类后，我们需要在 settings.py 中使用它们。MongoDB 的连接信息还需要定义。 我们在 settings.py 中加入如下内容：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">   <span class="string">'tutorial.pipelines.TextPipeline'</span>: 300,</span><br><span class="line">   <span class="string">'tutorial.pipelines.MongoPipeline'</span>: 400,</span><br><span class="line">&#125;</span><br><span class="line"><span class="attribute">MONGO_URI</span>=<span class="string">'localhost'</span></span><br><span class="line"><span class="attribute">MONGO_DB</span>=<span class="string">'tutorial'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>赋值 ITEM_PIPELINES 字典，键名是 Pipeline 的类名称，键值是调用优先级，是一个数字，数字越小则对应的 Pipeline 越先被调用。 再重新执行爬取，命令如下所示：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">scrapy crawl quotes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>爬取结束后，MongoDB 中创建了一个 tutorial 的数据库、QuoteItem 的表，如图 13-4 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-033919.jpg" alt=""> 图 13-4 爬取结果 长的 text 已经被处理并追加了省略号，短的 text 保持不变，author 和 tags 也都相应保存。</p>
                  <h3 id="12-源代码"><a href="#12-源代码" class="headerlink" title="12. 源代码"></a>12. 源代码</h3>
                  <p>本节代码地址：<a href="https://github.com/Python3WebSpider/ScrapyTutorial" target="_blank" rel="noopener">https://github.com/Python3WebSpider/ScrapyTutorial</a>。</p>
                  <h3 id="13-结语"><a href="#13-结语" class="headerlink" title="13. 结语"></a>13. 结语</h3>
                  <p>我们通过抓取 Quotes 网站完成了整个 Scrapy 的简单入门。但这只是冰山一角，还有很多内容等待我们去探索。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-02 11:22:46" itemprop="dateCreated datePublished" datetime="2019-12-02T11:22:46+08:00">2019-12-02</time>
                </span>
                <span id="/8337.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.2-Scrapy 入门" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>14k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>13 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8333.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Paper <i class="label-arrow"></i>
                  </a>
                  <a href="/8333.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 13.1-Scrapy 框架介绍</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="13-1-Scrapy-框架介绍"><a href="#13-1-Scrapy-框架介绍" class="headerlink" title="13.1 Scrapy 框架介绍"></a>13.1 Scrapy 框架介绍</h1>
                  <p>Scrapy 是一个基于 Twisted 的异步处理框架，是纯 Python 实现的爬虫框架，其架构清晰，模块之间的耦合程度低，可扩展性极强，可以灵活完成各种需求。我们只需要定制开发几个模块就可以轻松实现一个爬虫。</p>
                  <h3 id="1-架构介绍"><a href="#1-架构介绍" class="headerlink" title="1. 架构介绍"></a>1. 架构介绍</h3>
                  <p>首先我们来看下 Scrapy 框架的架构，如图 13-1 所示： <img src="https://qiniu.cuiqingcai.com/2019-11-27-033839.jpg" alt=""> 图 13-1 Scrapy 架构 它可以分为如下的几个部分。</p>
                  <ul>
                    <li>Engine，引擎，用来处理整个系统的数据流处理，触发事务，是整个框架的核心。</li>
                    <li>Item，项目，它定义了爬取结果的数据结构，爬取的数据会被赋值成该对象。</li>
                    <li>Scheduler， 调度器，用来接受引擎发过来的请求并加入队列中，并在引擎再次请求的时候提供给引擎。</li>
                    <li>Downloader，下载器，用于下载网页内容，并将网页内容返回给蜘蛛。</li>
                    <li>Spiders，蜘蛛，其内定义了爬取的逻辑和网页的解析规则，它主要负责解析响应并生成提取结果和新的请求。</li>
                    <li>Item Pipeline，项目管道，负责处理由蜘蛛从网页中抽取的项目，它的主要任务是清洗、验证和存储数据。</li>
                    <li>Downloader Middlewares，下载器中间件，位于引擎和下载器之间的钩子框架，主要是处理引擎与下载器之间的请求及响应。</li>
                    <li>Spider Middlewares， 蜘蛛中间件，位于引擎和蜘蛛之间的钩子框架，主要工作是处理蜘蛛输入的响应和输出的结果及新的请求。</li>
                  </ul>
                  <h3 id="2-数据流"><a href="#2-数据流" class="headerlink" title="2. 数据流"></a>2. 数据流</h3>
                  <p>Scrapy 中的数据流由引擎控制，其过程如下:</p>
                  <ul>
                    <li>Engine 首先打开一个网站，找到处理该网站的 Spider 并向该 Spider 请求第一个要爬取的 URL。</li>
                    <li>Engine 从 Spider 中获取到第一个要爬取的 URL 并通过 Scheduler 以 Request 的形式调度。</li>
                    <li>Engine 向 Scheduler 请求下一个要爬取的 URL。</li>
                    <li>Scheduler 返回下一个要爬取的 URL 给 Engine，Engine 将 URL 通过 Downloader Middlewares 转发给 Downloader 下载。</li>
                    <li>一旦页面下载完毕， Downloader 生成一个该页面的 Response，并将其通过 Downloader Middlewares 发送给 Engine。</li>
                    <li>Engine 从下载器中接收到 Response 并通过 Spider Middlewares 发送给 Spider 处理。</li>
                    <li>Spider 处理 Response 并返回爬取到的 Item 及新的 Request 给 Engine。</li>
                    <li>Engine 将 Spider 返回的 Item 给 Item Pipeline，将新的 Request 给 Scheduler。</li>
                    <li>重复第二步到最后一步，直到 Scheduler 中没有更多的 Request，Engine 关闭该网站，爬取结束。</li>
                  </ul>
                  <p>通过多个组件的相互协作、不同组件完成工作的不同、组件对异步处理的支持，Scrapy 最大限度地利用了网络带宽，大大提高了数据爬取和处理的效率。</p>
                  <h3 id="3-项目结构"><a href="#3-项目结构" class="headerlink" title="3. 项目结构"></a>3. 项目结构</h3>
                  <p>Scrapy 框架和 pyspider 不同，它是通过命令行来创建项目的，代码的编写还是需要 IDE。项目创建之后，项目文件结构如下所示：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">scrapy.cfg</span><br><span class="line">project/</span><br><span class="line">    __init__.py</span><br><span class="line">    items.py</span><br><span class="line">    pipelines.py</span><br><span class="line">    settings.py</span><br><span class="line">    middlewares.py</span><br><span class="line">    spiders/</span><br><span class="line">        __init__.py</span><br><span class="line">        spider1.py</span><br><span class="line">        spider2.py</span><br><span class="line">        ...</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在此要将各个文件的功能描述如下：</p>
                  <ul>
                    <li>scrapy.cfg：它是 Scrapy 项目的配置文件，其内定义了项目的配置文件路径、部署相关信息等内容。</li>
                    <li>items.py：它定义 Item 数据结构，所有的 Item 的定义都可以放这里。</li>
                    <li>pipelines.py：它定义 Item Pipeline 的实现，所有的 Item Pipeline 的实现都可以放这里。</li>
                    <li>settings.py：它定义项目的全局配置。</li>
                    <li>middlewares.py：它定义 Spider Middlewares 和 Downloader Middlewares 的实现。</li>
                    <li>spiders：其内包含一个个 Spider 的实现，每个 Spider 都有一个文件。</li>
                  </ul>
                  <h3 id="4-结语"><a href="#4-结语" class="headerlink" title="4. 结语"></a>4. 结语</h3>
                  <p>本节介绍了 Scrapy 框架的基本架构、数据流过程以及项目结构。后面我们会详细了解 Scrapy 的用法，感受它的强大。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2019-12-02 11:20:01" itemprop="dateCreated datePublished" datetime="2019-12-02T11:20:01+08:00">2019-12-02</time>
                </span>
                <span id="/8333.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.1-Scrapy 框架介绍" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/9/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/9/">9</a><span class="page-number current">10</span><a class="page-number" href="/page/11/">11</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/11/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
